Phase 9: application services and admin completion

Implements PLAN.md Phase 9. internal/appservice reads registration files
(appservice.config_files), matches namespaces the way Synapse does
(anchored at the start), and delivers events to each service as
transactions read from the roomserver's event stream at a cursor kept in
appservice_state. A transaction in flight is recorded, so after a restart
it is resent under the same ID; failures back off from 2s to 5m, and a
successful ping cuts the backoff short.

In the Client-Server API an as_token acts as the service's sender or, with
?user_id=, as any existing user in its namespace. Services register and
log in their users with m.login.application_service; exclusive user and
alias namespaces are refused to everybody else with M_EXCLUSIVE. Unknown
aliases and users in a namespace are asked of the service first. The
ping endpoint reports M_URL_NOT_SET, M_BAD_STATUS, M_CONNECTION_FAILED
and M_CONNECTION_TIMEOUT. Services' own requests, and their users' when
rate_limited is false, skip the rate limiter.

Users can change their password (POST /account/password) and deactivate
their account (POST /account/deactivate), both behind the password.
Deactivation revokes tokens, deletes devices, optionally erases the
profile and leaves every room through a new userapi.AccountObserver.

The admin API is complete: user detail, the admin flag, deactivation and
reactivation, password resets, devices, media quotas
(media.max_user_storage_bytes plus per-user overrides), registration
tokens with use limits and expiry (registration.require_token), server
notices (internal/servernotices, server_notices.*), an editor for the
federation allow/deny lists persisted in server_settings, a delivery view
of the application services, and an audit log of every change.

Migration 0011 adds the registration token, server notice room, media
quota, audit log and appservice cursor tables. Policy lists are now
returned sorted. The Complement entrypoint loads the registrations
Complement mounts in /complement/appservice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/storage/sqlstore/admin.go

@@ -0,0 +1,317 @@
+package sqlstore
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/dariuszjaniak/warp/server/internal/storage"
+)
+
+// The admin store's sequence table and its one counter.
+const (
+	adminSequences = "admin_sequences"
+	seqAudit       = "audit"
+)
+
+// Admin is the shared implementation of storage.AdminStore.
+type Admin struct {
+	db       *sql.DB
+	rebind   func(string) string
+	isUnique func(error) bool
+}
+
+// NewAdmin builds an admin store.
+func NewAdmin(db *sql.DB, rebind func(string) string, isUnique func(error) bool) *Admin {
+	if rebind == nil {
+		rebind = func(query string) string { return query }
+	}
+	if isUnique == nil {
+		isUnique = func(error) bool { return false }
+	}
+	return &Admin{db: db, rebind: rebind, isUnique: isUnique}
+}
+
+func (s *Admin) q(query string) string { return s.rebind(query) }
+
+const (
+	registrationTokenColumns = `token, uses_allowed, completed, expires_at_ms, created_at_ms`
+
+	insertRegistrationTokenSQL = `INSERT INTO admin_registration_tokens (` + registrationTokenColumns + `)
+		VALUES (?, ?, ?, ?, ?)`
+
+	selectRegistrationTokenSQL = `SELECT ` + registrationTokenColumns + `
+		FROM admin_registration_tokens WHERE token = ?`
+
+	selectRegistrationTokensSQL = `SELECT ` + registrationTokenColumns + `
+		FROM admin_registration_tokens ORDER BY token`
+
+	updateRegistrationTokenSQL = `UPDATE admin_registration_tokens
+		SET uses_allowed = ?, expires_at_ms = ? WHERE token = ?`
+
+	deleteRegistrationTokenSQL = `DELETE FROM admin_registration_tokens WHERE token = ?`
+
+	// The usability test is in the WHERE clause, so that checking and
+	// spending are one statement and two registrations cannot both take
+	// the last use.
+	useRegistrationTokenSQL = `UPDATE admin_registration_tokens SET completed = completed + 1
+		WHERE token = ?
+		AND (uses_allowed IS NULL OR completed < uses_allowed)
+		AND (expires_at_ms = 0 OR expires_at_ms > ?)`
+
+	releaseRegistrationTokenSQL = `UPDATE admin_registration_tokens SET completed = completed - 1
+		WHERE token = ? AND completed > 0`
+)
+
+// CreateRegistrationToken implements storage.AdminStore.
+func (s *Admin) CreateRegistrationToken(ctx context.Context, t storage.RegistrationToken) error {
+	_, err := s.db.ExecContext(ctx, s.q(insertRegistrationTokenSQL),
+		t.Token, nullableInt(t.UsesAllowed), t.Completed, t.ExpiresAtMS, t.CreatedAtMS)
+	switch {
+	case err != nil && s.isUnique(err):
+		return storage.ErrRegistrationTokenExists
+	case err != nil:
+		return fmt.Errorf("create a registration token: %w", err)
+	}
+	return nil
+}
+
+// RegistrationToken implements storage.AdminStore.
+func (s *Admin) RegistrationToken(ctx context.Context, token string) (storage.RegistrationToken, error) {
+	t, err := scanRegistrationToken(s.db.QueryRowContext(ctx, s.q(selectRegistrationTokenSQL), token).Scan)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return storage.RegistrationToken{}, storage.ErrNoSuchRegistrationToken
+	case err != nil:
+		return storage.RegistrationToken{}, fmt.Errorf("read a registration token: %w", err)
+	}
+	return t, nil
+}
+
+// RegistrationTokens implements storage.AdminStore.
+func (s *Admin) RegistrationTokens(ctx context.Context) ([]storage.RegistrationToken, error) {
+	rows, err := s.db.QueryContext(ctx, s.q(selectRegistrationTokensSQL))
+	if err != nil {
+		return nil, fmt.Errorf("list registration tokens: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	out := []storage.RegistrationToken{}
+	for rows.Next() {
+		t, err := scanRegistrationToken(rows.Scan)
+		if err != nil {
+			return nil, fmt.Errorf("list registration tokens: %w", err)
+		}
+		out = append(out, t)
+	}
+	return out, rows.Err()
+}
+
+// UpdateRegistrationToken implements storage.AdminStore.
+func (s *Admin) UpdateRegistrationToken(ctx context.Context, token string, usesAllowed *int, expiresAtMS int64) error {
+	res, err := s.db.ExecContext(ctx, s.q(updateRegistrationTokenSQL), nullableInt(usesAllowed), expiresAtMS, token)
+	if err != nil {
+		return fmt.Errorf("update a registration token: %w", err)
+	}
+	return mustHaveAffected(res, storage.ErrNoSuchRegistrationToken)
+}
+
+// DeleteRegistrationToken implements storage.AdminStore.
+func (s *Admin) DeleteRegistrationToken(ctx context.Context, token string) error {
+	res, err := s.db.ExecContext(ctx, s.q(deleteRegistrationTokenSQL), token)
+	if err != nil {
+		return fmt.Errorf("delete a registration token: %w", err)
+	}
+	return mustHaveAffected(res, storage.ErrNoSuchRegistrationToken)
+}
+
+// UseRegistrationToken implements storage.AdminStore.
+func (s *Admin) UseRegistrationToken(ctx context.Context, token string, nowMS int64) error {
+	res, err := s.db.ExecContext(ctx, s.q(useRegistrationTokenSQL), token, nowMS)
+	if err != nil {
+		return fmt.Errorf("spend a registration token: %w", err)
+	}
+	return mustHaveAffected(res, storage.ErrNoSuchRegistrationToken)
+}
+
+// ReleaseRegistrationToken implements storage.AdminStore. A token deleted
+// in the meantime has nothing to give back to, which is not an error.
+func (s *Admin) ReleaseRegistrationToken(ctx context.Context, token string) error {
+	if _, err := s.db.ExecContext(ctx, s.q(releaseRegistrationTokenSQL), token); err != nil {
+		return fmt.Errorf("release a registration token: %w", err)
+	}
+	return nil
+}
+
+func scanRegistrationToken(scan func(...any) error) (storage.RegistrationToken, error) {
+	var (
+		t    storage.RegistrationToken
+		uses sql.NullInt64
+	)
+	if err := scan(&t.Token, &uses, &t.Completed, &t.ExpiresAtMS, &t.CreatedAtMS); err != nil {
+		return storage.RegistrationToken{}, err
+	}
+	if uses.Valid {
+		n := int(uses.Int64)
+		t.UsesAllowed = &n
+	}
+	return t, nil
+}
+
+// nullableInt binds an optional count as NULL when it is absent.
+func nullableInt(n *int) any {
+	if n == nil {
+		return nil
+	}
+	return int64(*n)
+}
+
+// mustHaveAffected turns an UPDATE or DELETE that matched nothing into
+// notFound.
+func mustHaveAffected(res sql.Result, notFound error) error {
+	n, err := res.RowsAffected()
+	if err != nil {
+		return err
+	}
+	if n == 0 {
+		return notFound
+	}
+	return nil
+}
+
+const (
+	insertAuditSQL = `INSERT INTO admin_audit_log (id, ts_ms, actor, action, target, details_json)
+		VALUES (?, ?, ?, ?, ?, ?)`
+
+	auditColumns = `id, ts_ms, actor, action, target, details_json`
+
+	// defaultAuditPage and maxAuditPage bound a page of the audit log.
+	defaultAuditPage = 50
+	maxAuditPage     = 1000
+)
+
+// AppendAudit implements storage.AdminStore.
+func (s *Admin) AppendAudit(ctx context.Context, e storage.AuditEntry) (int64, error) {
+	details := string(e.Details)
+	if details == "" {
+		details = "{}"
+	}
+	var id int64
+	err := inTx(ctx, s.db, func(tx *sql.Tx) error {
+		var err error
+		if id, err = nextSequence(ctx, tx, s.q, adminSequences, seqAudit); err != nil {
+			return err
+		}
+		_, err = tx.ExecContext(ctx, s.q(insertAuditSQL), id, e.TSMS, e.Actor, e.Action, e.Target, details)
+		return err
+	})
+	if err != nil {
+		return 0, fmt.Errorf("append to the audit log: %w", err)
+	}
+	return id, nil
+}
+
+// AuditLog implements storage.AdminStore.
+func (s *Admin) AuditLog(ctx context.Context, f storage.AuditFilter) ([]storage.AuditEntry, error) {
+	var (
+		conditions []string
+		args       []any
+	)
+	if f.Before > 0 {
+		conditions = append(conditions, `id < ?`)
+		args = append(args, f.Before)
+	}
+	if f.Actor != "" {
+		conditions = append(conditions, `actor = ?`)
+		args = append(args, f.Actor)
+	}
+	if f.Action != "" {
+		conditions = append(conditions, `action = ?`)
+		args = append(args, f.Action)
+	}
+	limit := f.Limit
+	switch {
+	case limit <= 0:
+		limit = defaultAuditPage
+	case limit > maxAuditPage:
+		limit = maxAuditPage
+	}
+	args = append(args, limit)
+
+	query := `SELECT ` + auditColumns + ` FROM admin_audit_log`
+	if len(conditions) > 0 {
+		query += ` WHERE ` + strings.Join(conditions, " AND ")
+	}
+	query += ` ORDER BY id DESC LIMIT ?`
+
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the audit log: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	out := []storage.AuditEntry{}
+	for rows.Next() {
+		var (
+			e       storage.AuditEntry
+			details string
+		)
+		if err := rows.Scan(&e.ID, &e.TSMS, &e.Actor, &e.Action, &e.Target, &details); err != nil {
+			return nil, fmt.Errorf("read the audit log: %w", err)
+		}
+		e.Details = []byte(details)
+		out = append(out, e)
+	}
+	return out, rows.Err()
+}
+
+const (
+	selectServerNoticeRoomSQL = `SELECT room_id FROM admin_server_notice_rooms WHERE localpart = ?`
+
+	upsertServerNoticeRoomSQL = `
+		INSERT INTO admin_server_notice_rooms (localpart, room_id) VALUES (?, ?)
+		ON CONFLICT (localpart) DO UPDATE SET room_id = excluded.room_id`
+)
+
+// ServerNoticeRoom implements storage.AdminStore.
+func (s *Admin) ServerNoticeRoom(ctx context.Context, localpart string) (string, error) {
+	var roomID string
+	err := s.db.QueryRowContext(ctx, s.q(selectServerNoticeRoomSQL), localpart).Scan(&roomID)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return "", storage.ErrNoServerNoticeRoom
+	case err != nil:
+		return "", fmt.Errorf("read the server notice room of %q: %w", localpart, err)
+	}
+	return roomID, nil
+}
+
+// SetServerNoticeRoom implements storage.AdminStore.
+func (s *Admin) SetServerNoticeRoom(ctx context.Context, localpart, roomID string) error {
+	if _, err := s.db.ExecContext(ctx, s.q(upsertServerNoticeRoomSQL), localpart, roomID); err != nil {
+		return fmt.Errorf("record the server notice room of %q: %w", localpart, err)
+	}
+	return nil
+}
+
+// inTx runs fn in a transaction on db, rolling back on any error.
+func inTx(ctx context.Context, db *sql.DB, fn func(*sql.Tx) error) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	if err := fn(tx); err != nil {
+		return err
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit transaction: %w", err)
+	}
+	return nil
+}
+
+// Compile-time proof that the shared implementation satisfies the interface.
+var _ storage.AdminStore = (*Admin)(nil)

server/internal/storage/sqlstore/appservices.go

@@ -0,0 +1,70 @@
+package sqlstore
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/dariuszjaniak/warp/server/internal/storage"
+)
+
+// AppServices is the shared implementation of storage.AppServiceStore.
+type AppServices struct {
+	db     *sql.DB
+	rebind func(string) string
+}
+
+// NewAppServices builds an application service store.
+func NewAppServices(db *sql.DB, rebind func(string) string) *AppServices {
+	if rebind == nil {
+		rebind = func(query string) string { return query }
+	}
+	return &AppServices{db: db, rebind: rebind}
+}
+
+func (s *AppServices) q(query string) string { return s.rebind(query) }
+
+const (
+	insertAppServiceStateSQL = `
+		INSERT INTO appservice_state (as_id, stream_pos) VALUES (?, ?)
+		ON CONFLICT (as_id) DO NOTHING`
+
+	selectAppServiceStateSQL = `
+		SELECT as_id, stream_pos, txn_id, txn_to FROM appservice_state WHERE as_id = ?`
+
+	beginAppServiceTxnSQL = `UPDATE appservice_state SET txn_id = ?, txn_to = ? WHERE as_id = ?`
+
+	completeAppServiceTxnSQL = `UPDATE appservice_state SET stream_pos = ?, txn_to = 0 WHERE as_id = ?`
+)
+
+// AppServiceState implements storage.AppServiceStore.
+func (s *AppServices) AppServiceState(ctx context.Context, id string, startPos int64) (storage.AppServiceState, error) {
+	if _, err := s.db.ExecContext(ctx, s.q(insertAppServiceStateSQL), id, startPos); err != nil {
+		return storage.AppServiceState{}, fmt.Errorf("start the cursor of application service %q: %w", id, err)
+	}
+	var st storage.AppServiceState
+	if err := s.db.QueryRowContext(ctx, s.q(selectAppServiceStateSQL), id).
+		Scan(&st.ID, &st.StreamPos, &st.TxnID, &st.TxnTo); err != nil {
+		return storage.AppServiceState{}, fmt.Errorf("read the cursor of application service %q: %w", id, err)
+	}
+	return st, nil
+}
+
+// BeginTransaction implements storage.AppServiceStore.
+func (s *AppServices) BeginTransaction(ctx context.Context, id string, txnID, to int64) error {
+	if _, err := s.db.ExecContext(ctx, s.q(beginAppServiceTxnSQL), txnID, to, id); err != nil {
+		return fmt.Errorf("begin transaction %d of application service %q: %w", txnID, id, err)
+	}
+	return nil
+}
+
+// CompleteTransaction implements storage.AppServiceStore.
+func (s *AppServices) CompleteTransaction(ctx context.Context, id string, to int64) error {
+	if _, err := s.db.ExecContext(ctx, s.q(completeAppServiceTxnSQL), to, id); err != nil {
+		return fmt.Errorf("advance the cursor of application service %q: %w", id, err)
+	}
+	return nil
+}
+
+// Compile-time proof that the shared implementation satisfies the interface.
+var _ storage.AppServiceStore = (*AppServices)(nil)

server/internal/storage/sqlstore/media.go

@@ -172,3 +172,61 @@ func (s *Media) StoreThumbnail(ctx context.Context, t storage.Thumbnail) error {
 	}
 	return nil
 }
+
+const selectMediaUsageSQL = `
+	SELECT COUNT(*), COALESCE(SUM(file_size), 0) FROM media_repository
+	WHERE user_id = ? AND media_origin = ? AND pending = ?`
+
+// MediaUsage implements storage.MediaStore.
+//
+// It counts every upload at its own size, even when two of them are the
+// same bytes and the blob store holds one copy: a quota is about what a
+// user asked this server to keep, and how the server keeps it is its own
+// business.
+func (s *Media) MediaUsage(ctx context.Context, origin, userID string) (storage.MediaUsage, error) {
+	var u storage.MediaUsage
+	if err := s.db.QueryRowContext(ctx, s.q(selectMediaUsageSQL), userID, origin, false).
+		Scan(&u.Count, &u.Bytes); err != nil {
+		return storage.MediaUsage{}, fmt.Errorf("total the uploads of %s: %w", userID, err)
+	}
+	return u, nil
+}
+
+const (
+	selectMediaQuotaSQL = `SELECT max_bytes FROM media_quotas WHERE user_id = ?`
+
+	upsertMediaQuotaSQL = `
+		INSERT INTO media_quotas (user_id, max_bytes) VALUES (?, ?)
+		ON CONFLICT (user_id) DO UPDATE SET max_bytes = excluded.max_bytes`
+
+	deleteMediaQuotaSQL = `DELETE FROM media_quotas WHERE user_id = ?`
+)
+
+// MediaQuota implements storage.MediaStore.
+func (s *Media) MediaQuota(ctx context.Context, userID string) (int64, bool, error) {
+	var maxBytes int64
+	err := s.db.QueryRowContext(ctx, s.q(selectMediaQuotaSQL), userID).Scan(&maxBytes)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return 0, false, nil
+	case err != nil:
+		return 0, false, fmt.Errorf("read the media quota of %s: %w", userID, err)
+	}
+	return maxBytes, true, nil
+}
+
+// SetMediaQuota implements storage.MediaStore.
+func (s *Media) SetMediaQuota(ctx context.Context, userID string, maxBytes int64) error {
+	if _, err := s.db.ExecContext(ctx, s.q(upsertMediaQuotaSQL), userID, maxBytes); err != nil {
+		return fmt.Errorf("set the media quota of %s: %w", userID, err)
+	}
+	return nil
+}
+
+// DeleteMediaQuota implements storage.MediaStore.
+func (s *Media) DeleteMediaQuota(ctx context.Context, userID string) error {
+	if _, err := s.db.ExecContext(ctx, s.q(deleteMediaQuotaSQL), userID); err != nil {
+		return fmt.Errorf("delete the media quota of %s: %w", userID, err)
+	}
+	return nil
+}

server/internal/storage/sqlstore/sqlstore.go

@@ -63,6 +63,8 @@ type Store struct {
 	media    *Media
 	keys     *Keys
 	push     *Push
+	admin    *Admin
+	apps     *AppServices
 }
 
 // New builds a Store. It does not touch the database.
@@ -85,6 +87,8 @@ func New(opts Options) (*Store, error) {
 		media:    NewMedia(opts.DB, opts.Rebind, opts.IsUniqueViolation),
 		keys:     NewKeys(opts.DB, opts.Rebind),
 		push:     NewPush(opts.DB, opts.Rebind),
+		admin:    NewAdmin(opts.DB, opts.Rebind, opts.IsUniqueViolation),
+		apps:     NewAppServices(opts.DB, opts.Rebind),
 	}, nil
 }
 
@@ -124,6 +128,12 @@ func (s *Store) Keys() storage.KeyStore { return s.keys }
 // Push implements storage.Database.
 func (s *Store) Push() storage.PushStore { return s.push }
 
+// Admin implements storage.Database.
+func (s *Store) Admin() storage.AdminStore { return s.admin }
+
+// AppServices implements storage.Database.
+func (s *Store) AppServices() storage.AppServiceStore { return s.apps }
+
 // Migrate implements storage.Database.
 func (s *Store) Migrate(ctx context.Context) error {
 	ms, err := s.migrations()

server/internal/storage/sqlstore/timeline.go

@@ -142,6 +142,40 @@ func (s *Rooms) Timeline(ctx context.Context, req storage.TimelineRequest) ([]st
 	return events, nil
 }
 
+// StreamEvents implements storage.RoomStore.
+//
+// It reads by stream_ordering alone, which the column's own uniqueness
+// index serves. Backfilled history sits below zero and a reader starting at
+// a real position never reaches it, which is right for what this is for:
+// an application service is told what happens, not what a backfill dug up.
+func (s *Rooms) StreamEvents(ctx context.Context, after, upTo int64, limit int) ([]storage.StoredEvent, error) {
+	if limit <= 0 {
+		return nil, nil
+	}
+	conditions := []string{`e.stream_ordering > ?`, `NOT e.outlier`}
+	args := []any{after}
+	if upTo != 0 {
+		conditions = append(conditions, `e.stream_ordering <= ?`)
+		args = append(args, upTo)
+	}
+	args = append(args, limit)
+
+	query := `SELECT ` + eventColumns + `
+		FROM roomserver_events e
+		WHERE ` + strings.Join(conditions, " AND ") + `
+		ORDER BY e.stream_ordering
+		LIMIT ?`
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the event stream after %d: %w", after, err)
+	}
+	events, err := scanEvents(rows)
+	if err != nil {
+		return nil, fmt.Errorf("read the event stream after %d: %w", after, err)
+	}
+	return events, nil
+}
+
 // filterConditions renders the part of an event filter the store applies in
 // SQL, as a WHERE fragment and its arguments.
 //

server/internal/storage/sqlstore/users.go

@@ -140,6 +140,12 @@ func (u *Users) SetAdmin(ctx context.Context, localpart string, admin bool) erro
 		`UPDATE user_accounts SET admin = ? WHERE localpart = ?`, admin)
 }
 
+// SetDeactivated implements storage.UserStore.
+func (u *Users) SetDeactivated(ctx context.Context, localpart string, deactivated bool) error {
+	return u.updateAccount(ctx, localpart,
+		`UPDATE user_accounts SET deactivated = ? WHERE localpart = ?`, deactivated)
+}
+
 // SetDisplayName implements storage.UserStore.
 func (u *Users) SetDisplayName(ctx context.Context, localpart, displayName string) error {
 	return u.updateAccount(ctx, localpart,