Phase 9: application services and admin completion

Implements PLAN.md Phase 9. internal/appservice reads registration files
(appservice.config_files), matches namespaces the way Synapse does
(anchored at the start), and delivers events to each service as
transactions read from the roomserver's event stream at a cursor kept in
appservice_state. A transaction in flight is recorded, so after a restart
it is resent under the same ID; failures back off from 2s to 5m, and a
successful ping cuts the backoff short.

In the Client-Server API an as_token acts as the service's sender or, with
?user_id=, as any existing user in its namespace. Services register and
log in their users with m.login.application_service; exclusive user and
alias namespaces are refused to everybody else with M_EXCLUSIVE. Unknown
aliases and users in a namespace are asked of the service first. The
ping endpoint reports M_URL_NOT_SET, M_BAD_STATUS, M_CONNECTION_FAILED
and M_CONNECTION_TIMEOUT. Services' own requests, and their users' when
rate_limited is false, skip the rate limiter.

Users can change their password (POST /account/password) and deactivate
their account (POST /account/deactivate), both behind the password.
Deactivation revokes tokens, deletes devices, optionally erases the
profile and leaves every room through a new userapi.AccountObserver.

The admin API is complete: user detail, the admin flag, deactivation and
reactivation, password resets, devices, media quotas
(media.max_user_storage_bytes plus per-user overrides), registration
tokens with use limits and expiry (registration.require_token), server
notices (internal/servernotices, server_notices.*), an editor for the
federation allow/deny lists persisted in server_settings, a delivery view
of the application services, and an audit log of every change.

Migration 0011 adds the registration token, server notice room, media
quota, audit log and appservice cursor tables. Policy lists are now
returned sorted. The Complement entrypoint loads the registrations
Complement mounts in /complement/appservice.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/setup/server.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/dariuszjaniak/warp/server/internal/adminapi"
+	"github.com/dariuszjaniak/warp/server/internal/appservice"
 	"github.com/dariuszjaniak/warp/server/internal/buildinfo"
 	"github.com/dariuszjaniak/warp/server/internal/clientapi"
 	"github.com/dariuszjaniak/warp/server/internal/config"
@@ -27,6 +28,7 @@ import (
 	"github.com/dariuszjaniak/warp/server/internal/metrics"
 	"github.com/dariuszjaniak/warp/server/internal/pushserver"
 	"github.com/dariuszjaniak/warp/server/internal/roomserver"
+	"github.com/dariuszjaniak/warp/server/internal/servernotices"
 	"github.com/dariuszjaniak/warp/server/internal/settings"
 	"github.com/dariuszjaniak/warp/server/internal/storage"
 	_ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" // link in the storage drivers
@@ -52,7 +54,12 @@ type Server struct {
 	push      *pushserver.API
 	media     *mediaapi.API
 	clientAPI *clientapi.ClientAPI
-	adminAPI  *adminapi.AdminAPI
+
+	appservices *appservice.API
+	notices     *servernotices.API
+	regTokens   *userapi.RegistrationTokens
+
+	adminAPI *adminapi.AdminAPI
 
 	// The federation components. All nil when the server started without
 	// a signing key, because nothing that federates can work without one.
@@ -151,6 +158,8 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 		FederationAllowRuntimeToggle:   cfg.Federation.AllowRuntimeToggle,
 		RegistrationEnabled:            cfg.Registration.Enabled,
 		RegistrationAllowRuntimeToggle: cfg.Registration.AllowRuntimeToggle,
+		FederationAllowlist:            cfg.Federation.Allowlist,
+		FederationDenylist:             cfg.Federation.Denylist,
 	}, log)
 	if err != nil {
 		_ = s.closeDB()
@@ -158,9 +167,12 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 	}
 	s.settings = settingsService
 
+	// The lists in force are the settings service's: the config file's,
+	// unless an admin has replaced them through the admin API.
+	lists := settingsService.FederationLists()
 	s.federation = federation.NewGate(
 		settingsService.FederationEnabled,
-		federation.NewPolicy(cfg.Federation.Allowlist, cfg.Federation.Denylist),
+		federation.NewPolicy(lists.Allowlist, lists.Denylist),
 	)
 	// The switch starts and stops the federation module. The gate and the
 	// client read the switch on every request and need no telling.
@@ -192,6 +204,29 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 		return nil, err
 	}
 
+	// Registration tokens come from the config file and from the admin
+	// API; the sign-up flow and the admin endpoints share one view of both.
+	s.regTokens = userapi.NewRegistrationTokens(userapi.RegistrationTokensOptions{
+		Store:    s.db.Admin(),
+		Static:   cfg.Registration.Tokens,
+		Required: cfg.Registration.RequireToken,
+		NowMS:    func() int64 { return time.Now().UnixMilli() },
+	})
+
+	// Application services are read from their registration files now,
+	// so that a bad one stops the server before it listens. Their senders
+	// are created here too: a service acts as its sender from its first
+	// request.
+	registry, err := loadAppServices(cfg)
+	if err != nil {
+		_ = s.closeDB()
+		return nil, err
+	}
+	if err := ensureAppServiceSenders(ctx, s.users, registry, log); err != nil {
+		_ = s.closeDB()
+		return nil, err
+	}
+
 	// The media repository needs no key: uploads and downloads of local
 	// media are not signed. Fetching other servers' media is, and is
 	// handed the federation client below once there is one.
@@ -342,6 +377,47 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 		}
 		s.sync.SetOutbound(federationapi.NewEDUs(queue, s.rooms))
 
+		// Application services read the event stream from a cursor of
+		// their own, and only need waking when it moves: they are one
+		// more listener on the emit stage.
+		s.appservices, err = appservice.New(appservice.Options{
+			Registry: registry,
+			Store:    s.db.AppServices(),
+			Rooms:    s.rooms,
+			Timeout:  cfg.AppService.Timeout,
+			Logger:   log,
+		})
+		if err != nil {
+			_ = s.closeDB()
+			return nil, err
+		}
+		emit.add(s.appservices)
+
+		// Deactivating an account leaves its rooms, which only the
+		// roomserver can do.
+		s.users.SetAccountObserver(roomLeaver{rooms: s.rooms, log: log})
+
+		// Server notices are room messages, so they come with the rooms.
+		if cfg.ServerNotices.Enabled {
+			s.notices, err = servernotices.New(servernotices.Options{
+				Localpart:   cfg.ServerNotices.Localpart,
+				DisplayName: cfg.ServerNotices.DisplayName,
+				RoomName:    cfg.ServerNotices.RoomName,
+				Accounts:    s.users,
+				Rooms:       s.rooms,
+				Store:       s.db.Admin(),
+				WakeUser:    notifier.NotifyUser,
+				Logger:      log,
+			})
+			if err == nil {
+				err = s.notices.EnsureUser(ctx)
+			}
+			if err != nil {
+				_ = s.closeDB()
+				return nil, err
+			}
+		}
+
 		s.fedAPI, err = federationapi.NewServer(federationapi.ServerOptions{
 			ServerName: cfg.Global.ServerName,
 			Gate:       s.federation,
@@ -360,6 +436,16 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 		}
 	} else {
 		log.Warn("started without a signing key: rooms, /sync and end-to-end encryption are unavailable")
+		// Without rooms there is no stream to deliver; the registry still
+		// lets the services use the Client-Server API.
+		if s.appservices, err = appservice.New(appservice.Options{
+			Registry: registry,
+			Timeout:  cfg.AppService.Timeout,
+			Logger:   log,
+		}); err != nil {
+			_ = s.closeDB()
+			return nil, err
+		}
 	}
 
 	// The interfaces below are left nil rather than set to a nil pointer
@@ -386,6 +472,8 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 		Federation:           s.federation,
 		RemoteProfiles:       remoteProfiles,
 		Registration:         cfg.Registration,
+		RegistrationTokens:   s.regTokens,
+		AppServices:          s.appservices,
 		RateLimit:            cfg.ClientAPI.RateLimit,
 		TrustXForwardedFor:   cfg.ClientAPI.TrustXForwardedFor,
 		Logger:               log,
@@ -396,20 +484,30 @@ func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error)
 	}
 
 	s.adminAPI, err = adminapi.New(adminapi.Options{
-		ServerName:       cfg.Global.ServerName,
-		Users:            s.users,
-		Rooms:            s.rooms,
-		Settings:         settingsService,
-		Federation:       s.federation,
-		FederationStatus: fedStatus,
-		Database:         s.db,
-		Logger:           log,
+		ServerName:         cfg.Global.ServerName,
+		Users:              s.users,
+		Rooms:              s.rooms,
+		Settings:           settingsService,
+		Federation:         s.federation,
+		FederationStatus:   fedStatus,
+		Database:           s.db,
+		Media:              s.media,
+		AppServices:        s.appservices,
+		Notices:            s.notices,
+		RegistrationTokens: s.regTokens,
+		Logger:             log,
 	})
 	if err != nil {
 		_ = s.closeDB()
 		return nil, err
 	}
 
+	// Delivery to application services starts with the server rather
+	// than with Run, like the push worker: it only reads the database and
+	// talks to the services, and tests that drive the handler directly
+	// need it running.
+	s.appservices.Start(ctx)
+
 	s.handler = httputil.Chain(s.routes(),
 		httputil.Recoverer,
 		httputil.RequestLogger(log),
@@ -477,6 +575,13 @@ func (s *Server) Keys() *keyserver.API { return s.keys }
 // signing key.
 func (s *Server) Push() *pushserver.API { return s.push }
 
+// AppServices exposes the application service component.
+func (s *Server) AppServices() *appservice.API { return s.appservices }
+
+// ServerNotices exposes the server notice sender, or nil when server
+// notices are off or the server has no signing key.
+func (s *Server) ServerNotices() *servernotices.API { return s.notices }
+
 // Media exposes the media repository.
 func (s *Server) Media() *mediaapi.API { return s.media }
 
@@ -624,6 +729,11 @@ func (s *Server) Close() error {
 		// to close.
 		s.push.Close()
 	}
+	if s.appservices != nil {
+		// So do the application service workers, which move their
+		// cursors.
+		s.appservices.Close()
+	}
 	return s.closeDB()
 }