Warp/server/internal/setup/server.go

// Package setup wires the components together into a running server: it
// opens storage, loads the signing key, builds the runtime-settings service
// and the HTTP stack, and owns the graceful shutdown.
package setup

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"log/slog"
	"net"
	"net/http"
	"sync/atomic"
	"time"

	"github.com/dariuszjaniak/warp/server/internal/adminapi"
	"github.com/dariuszjaniak/warp/server/internal/appservice"
	"github.com/dariuszjaniak/warp/server/internal/buildinfo"
	"github.com/dariuszjaniak/warp/server/internal/clientapi"
	"github.com/dariuszjaniak/warp/server/internal/config"
	"github.com/dariuszjaniak/warp/server/internal/federation"
	"github.com/dariuszjaniak/warp/server/internal/federationapi"
	"github.com/dariuszjaniak/warp/server/internal/httputil"
	"github.com/dariuszjaniak/warp/server/internal/keyserver"
	"github.com/dariuszjaniak/warp/server/internal/matrix/signing"
	"github.com/dariuszjaniak/warp/server/internal/mediaapi"
	"github.com/dariuszjaniak/warp/server/internal/metrics"
	"github.com/dariuszjaniak/warp/server/internal/pushserver"
	"github.com/dariuszjaniak/warp/server/internal/roomserver"
	"github.com/dariuszjaniak/warp/server/internal/servernotices"
	"github.com/dariuszjaniak/warp/server/internal/settings"
	"github.com/dariuszjaniak/warp/server/internal/storage"
	_ "github.com/dariuszjaniak/warp/server/internal/storage/drivers" // link in the storage drivers
	"github.com/dariuszjaniak/warp/server/internal/syncapi"
	"github.com/dariuszjaniak/warp/server/internal/tracing"
	"github.com/dariuszjaniak/warp/server/internal/userapi"
)

// Server is a configured, not yet listening homeserver.
type Server struct {
	cfg *config.Config
	log *slog.Logger

	db         storage.Database
	settings   *settings.Service
	metrics    *metrics.Registry
	federation *federation.Gate
	signingKey *signing.KeyPair

	// stopTracing flushes and stops the tracer provider tracing.Setup
	// installed.
	stopTracing func(context.Context) error

	users     *userapi.API
	rooms     *roomserver.API
	sync      *syncapi.API
	keys      *keyserver.API
	push      *pushserver.API
	media     *mediaapi.API
	clientAPI *clientapi.ClientAPI

	appservices *appservice.API
	notices     *servernotices.API
	regTokens   *userapi.RegistrationTokens

	adminAPI *adminapi.AdminAPI

	// The federation components. All nil when the server started without
	// a signing key, because nothing that federates can work without one.
	fedClient *federation.Client
	keyRing   *federation.KeyRing
	fedModule *federationapi.Module
	fedAPI    *federationapi.Server

	// tlsConfig serves the federation listener; nil when there is none.
	tlsConfig *tls.Config

	// running is set while Run is serving. The federation switch only
	// starts and stops the module while it is: a server that was built and
	// never run must not leave a queue running behind it.
	running atomic.Bool

	handler http.Handler

	// addr is the address Run is listening on; empty until then.
	addr atomic.Pointer[string]

	// fedAddr is the TLS federation listener's address while Run is
	// serving one; empty otherwise.
	fedAddr atomic.Pointer[string]

	// ownsDB is false when the caller supplied the database and will close
	// it itself.
	ownsDB bool
}

// Options are the knobs tests and commands need beyond the config file.
type Options struct {
	// Logger receives everything the server logs.
	Logger *slog.Logger

	// Database, when set, is used instead of opening one from the config.
	// Tests pass an in-memory SQLite database this way.
	Database storage.Database

	// SkipSigningKey starts the server without loading the signing key.
	// Only tests and `warp migrate` use it: a real server must be able to
	// sign.
	SkipSigningKey bool

	// FederationResolve replaces server discovery, and FederationTLS the
	// outbound TLS configuration. Tests that run several servers in one
	// process use them to point server names at local listeners whose
	// certificates they trust.
	FederationResolve federation.ResolveFunc
	FederationTLS     *tls.Config
}

// New builds the server: storage, migrations, settings, routes. It does not
// listen; call Run for that.
func New(ctx context.Context, cfg *config.Config, opts Options) (*Server, error) {
	log := opts.Logger
	if log == nil {
		log = slog.Default()
	}

	s := &Server{cfg: cfg, log: log, metrics: metrics.NewRegistry()}

	stopTracing, err := tracing.Setup(ctx, tracing.Config{
		Enabled:        cfg.Tracing.Enabled,
		Endpoint:       cfg.Tracing.Endpoint,
		SampleRatio:    cfg.Tracing.SampleRatio,
		ServiceName:    cfg.Tracing.ServiceName,
		ServiceVersion: buildinfo.Version,
		ServerName:     cfg.Global.ServerName,
	})
	if err != nil {
		return nil, err
	}
	s.stopTracing = stopTracing

	if opts.Database != nil {
		s.db = opts.Database
	} else {
		db, err := storage.Open(ctx, cfg.Database.Storage(), log)
		if err != nil {
			return nil, err
		}
		s.db, s.ownsDB = db, true
	}
	s.registerDBMetrics()

	if err := s.prepareSchema(ctx); err != nil {
		_ = s.closeDB()
		return nil, err
	}

	if !opts.SkipSigningKey {
		key, err := signing.Load(cfg.Global.PrivateKeyPath)
		if err != nil {
			_ = s.closeDB()
			return nil, fmt.Errorf("%w\nrun `warp generate-keys --config <file>` to create one", err)
		}
		s.signingKey = key
		log.Info("loaded signing key", "key_id", key.KeyID(), "path", cfg.Global.PrivateKeyPath)
	}

	settingsService, err := settings.New(ctx, s.db.Settings(), settings.Defaults{
		FederationEnabled:              cfg.Federation.Enabled,
		FederationAllowRuntimeToggle:   cfg.Federation.AllowRuntimeToggle,
		RegistrationEnabled:            cfg.Registration.Enabled,
		RegistrationAllowRuntimeToggle: cfg.Registration.AllowRuntimeToggle,
		FederationAllowlist:            cfg.Federation.Allowlist,
		FederationDenylist:             cfg.Federation.Denylist,
	}, log)
	if err != nil {
		_ = s.closeDB()
		return nil, err
	}
	s.settings = settingsService

	// The lists in force are the settings service's: the config file's,
	// unless an admin has replaced them through the admin API.
	lists := settingsService.FederationLists()
	s.federation = federation.NewGate(
		settingsService.FederationEnabled,
		federation.NewPolicy(lists.Allowlist, lists.Denylist),
	)
	// The switch starts and stops the federation module. The gate and the
	// client read the switch on every request and need no telling.
	settingsService.Subscribe(func(snap settings.Snapshot) {
		log.Info("federation switch applied", "enabled", snap.FederationEnabled)
		if s.fedModule != nil && s.running.Load() {
			s.fedModule.Apply(context.Background(), snap.FederationEnabled)
		}
	})

	if cfg.Listen.FederationAddress != "" {
		cert, err := tls.LoadX509KeyPair(cfg.Listen.TLSCertFile, cfg.Listen.TLSKeyFile)
		if err != nil {
			_ = s.closeDB()
			return nil, fmt.Errorf("load the federation listener's certificate: %w", err)
		}
		s.tlsConfig = &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}
	}

	s.users, err = userapi.New(userapi.Options{
		ServerName:           cfg.Global.ServerName,
		Store:                s.db.Users(),
		AccessTokenLifetime:  cfg.ClientAPI.AccessTokenLifetime,
		RefreshTokenLifetime: cfg.ClientAPI.RefreshTokenLifetime,
		Logger:               log,
	})
	if err != nil {
		_ = s.closeDB()
		return nil, err
	}

	// Registration tokens come from the config file and from the admin
	// API; the sign-up flow and the admin endpoints share one view of both.
	s.regTokens = userapi.NewRegistrationTokens(userapi.RegistrationTokensOptions{
		Store:    s.db.Admin(),
		Static:   cfg.Registration.Tokens,
		Required: cfg.Registration.RequireToken,
		NowMS:    func() int64 { return time.Now().UnixMilli() },
	})

	// Application services are read from their registration files now,
	// so that a bad one stops the server before it listens. Their senders
	// are created here too: a service acts as its sender from its first
	// request.
	registry, err := loadAppServices(cfg)
	if err != nil {
		_ = s.closeDB()
		return nil, err
	}
	if err := ensureAppServiceSenders(ctx, s.users, registry, log); err != nil {
		_ = s.closeDB()
		return nil, err
	}

	// The media repository needs no key: uploads and downloads of local
	// media are not signed. Fetching other servers' media is, and is
	// handed the federation client below once there is one.
	s.media, err = newMedia(cfg, s.db, log)
	if err != nil {
		_ = s.closeDB()
		return nil, err
	}

	// The roomserver needs the signing key: it is the only thing in Warp
	// that creates an event, and an event has to be signed. `warp migrate`
	// and the Phase 1 tests start without a key, and then start without
	// rooms — the C-S room endpoints answer M_UNRECOGNIZED rather than
	// pretending to work.
	//
	// The syncapi comes with it, because a sync is almost entirely room
	// reads: a server with no rooms has no stream to read, and /sync
	// answering M_UNRECOGNIZED says so where an empty success would not.
	//
	// The notifier is built first and handed to both: it is the
	// roomserver's emit stage and the syncapi's wake-up, which is exactly
	// the seam Phase 3 left for this.
	//
	// Federation hangs off the same key. The client and the key ring are
	// built whether or not the switch is on — they refuse everything while
	// it is off — and the outbound queue is a second listener on the
	// roomserver's emit stage next to the notifier.
	if s.signingKey != nil {
		notifier := syncapi.NewNotifier()
		emit := &fanout{}
		emit.add(notifier)
		emit.add(s.registerServerMetrics(notifier))

		clientTLS := opts.FederationTLS
		if clientTLS == nil {
			if clientTLS, err = federationTLS(cfg.Federation.TLSCAFile); err != nil {
				_ = s.closeDB()
				return nil, err
			}
		}
		s.fedClient = federation.NewClient(federation.ClientOptions{
			ServerName: cfg.Global.ServerName,
			Key:        s.signingKey,
			Resolve:    opts.FederationResolve,
			TLSConfig:  clientTLS,
			Enabled:    settingsService.FederationEnabled,
			Policy:     s.federation.Policy(),
		})
		s.media.SetRemote(s.fedClient)
		s.keyRing = federation.NewKeyRing(federation.KeyRingOptions{
			ServerName: cfg.Global.ServerName,
			Key:        s.signingKey,
			Store:      s.db.Federation(),
			Client:     s.fedClient,
			Notaries:   cfg.Federation.TrustedKeyServers,
			Validity:   cfg.Federation.KeyValidityPeriod,
			Logger:     log,
		})

		s.rooms, err = roomserver.New(roomserver.Options{
			ServerName:        cfg.Global.ServerName,
			Store:             s.db.Rooms(),
			Profiles:          s.users,
			SigningKey:        s.signingKey,
			FederationEnabled: settingsService.FederationEnabled,
			Notifier:          emit,
			Remote:            s.fedClient,
			Keys:              s.keyRing,
			Logger:            log,
		})
		if err != nil {
			_ = s.closeDB()
			return nil, err
		}

		queue := federationapi.NewQueue(federationapi.QueueOptions{
			ServerName: cfg.Global.ServerName,
			Store:      s.db.Federation(),
			Sender:     s.fedClient,
			Rooms:      s.rooms,
			Policy:     s.federation.Policy(),
			Logger:     log,
		})
		emit.add(queue)
		s.fedModule = federationapi.NewModule(queue, log)

		// The keyserver comes with the rooms too: device list tracking is
		// "who shares a room with whom", and the sync it reports through
		// is the roomserver's. It tells other servers about key changes
		// through the same queue as everything else, and hears about
		// deleted devices from the userapi.
		s.keys, err = keyserver.New(keyserver.Options{
			ServerName: cfg.Global.ServerName,
			Store:      s.db.Keys(),
			Rooms:      s.rooms,
			Notifier:   notifier,
			Devices:    s.users,
			Remote:     s.fedClient,
			Outbound:   queue,
			Logger:     log,
		})
		if err != nil {
			_ = s.closeDB()
			return nil, err
		}
		emit.add(keyMemberships{keys: s.keys})

		// The push server is the last listener on the emit stage. It
		// evaluates every event against the push rules of every local
		// member of its room, on a worker of its own, and it is what the
		// sync reads the unread counts from. Logging a device out takes
		// its pushers with it, which makes the userapi's device observer
		// two of them.
		s.push, err = pushserver.New(pushserver.Options{
			ServerName:            cfg.Global.ServerName,
			Store:                 s.db.Push(),
			Accounts:              s.users,
			Rooms:                 s.rooms,
			Notifier:              notifier,
			PushEnabled:           cfg.Push.Enabled,
			IncludeContent:        cfg.Push.IncludeContent,
			GatewayTimeout:        cfg.Push.GatewayTimeout,
			NotificationRetention: cfg.Push.NotificationRetention,
			Logger:                log,
		})
		if err != nil {
			_ = s.closeDB()
			return nil, err
		}
		emit.add(s.push)
		s.users.SetDeviceObserver(deviceObservers{s.keys, s.push})

		s.sync, err = syncapi.New(syncapi.Options{
			ServerName:      cfg.Global.ServerName,
			Rooms:           s.rooms,
			Accounts:        s.users,
			Keys:            s.keys,
			Push:            s.push,
			Store:           s.db.Sync(),
			Notifier:        notifier,
			PresenceEnabled: cfg.Sync.Presence,
			TypingTimeout:   cfg.Sync.TypingTimeout,
			LongPollTimeout: cfg.Sync.LongPollTimeout,
			Logger:          log,
		})
		if err != nil {
			_ = s.closeDB()
			return nil, err
		}
		s.sync.SetOutbound(federationapi.NewEDUs(queue, s.rooms))

		// Application services read the event stream from a cursor of
		// their own, and only need waking when it moves: they are one
		// more listener on the emit stage.
		s.appservices, err = appservice.New(appservice.Options{
			Registry: registry,
			Store:    s.db.AppServices(),
			Rooms:    s.rooms,
			Timeout:  cfg.AppService.Timeout,
			Logger:   log,
		})
		if err != nil {
			_ = s.closeDB()
			return nil, err
		}
		emit.add(s.appservices)

		// Deactivating an account leaves its rooms, which only the
		// roomserver can do.
		s.users.SetAccountObserver(roomLeaver{rooms: s.rooms, log: log})

		// Server notices are room messages, so they come with the rooms.
		if cfg.ServerNotices.Enabled {
			s.notices, err = servernotices.New(servernotices.Options{
				Localpart:   cfg.ServerNotices.Localpart,
				DisplayName: cfg.ServerNotices.DisplayName,
				RoomName:    cfg.ServerNotices.RoomName,
				Accounts:    s.users,
				Rooms:       s.rooms,
				Store:       s.db.Admin(),
				WakeUser:    notifier.NotifyUser,
				Logger:      log,
			})
			if err == nil {
				err = s.notices.EnsureUser(ctx)
			}
			if err != nil {
				_ = s.closeDB()
				return nil, err
			}
		}

		s.fedAPI, err = federationapi.NewServer(federationapi.ServerOptions{
			ServerName: cfg.Global.ServerName,
			Gate:       s.federation,
			Keys:       s.keyRing,
			Rooms:      s.rooms,
			Ephemeral:  s.sync,
			Profiles:   s.users,
			Store:      s.db.Federation(),
			Media:      s.media,
			UserKeys:   s.keys,
			Logger:     log,
		})
		if err != nil {
			_ = s.closeDB()
			return nil, err
		}
	} else {
		log.Warn("started without a signing key: rooms, /sync and end-to-end encryption are unavailable")
		// Without rooms there is no stream to deliver; the registry still
		// lets the services use the Client-Server API.
		if s.appservices, err = appservice.New(appservice.Options{
			Registry: registry,
			Timeout:  cfg.AppService.Timeout,
			Logger:   log,
		}); err != nil {
			_ = s.closeDB()
			return nil, err
		}
	}

	// The interfaces below are left nil rather than set to a nil pointer
	// when there is no federation: a nil *federation.Client in an
	// interface is not a nil interface, and would be called.
	var remoteProfiles clientapi.RemoteProfiles
	var fedStatus adminapi.FederationStatus
	if s.fedClient != nil {
		remoteProfiles = s.fedClient
		fedStatus = federationStatus{module: s.fedModule}
	}

	s.clientAPI, err = clientapi.New(clientapi.Options{
		ServerName:           cfg.Global.ServerName,
		ClientBaseURL:        cfg.ClientBaseURL(),
		Users:                s.users,
		Rooms:                s.rooms,
		Sync:                 s.sync,
		Keys:                 s.keys,
		Media:                s.media,
		Push:                 s.push,
		LegacyMediaDownloads: cfg.Media.LegacyUnauthenticatedDownloads,
		Settings:             settingsService,
		Federation:           s.federation,
		RemoteProfiles:       remoteProfiles,
		Registration:         cfg.Registration,
		RegistrationTokens:   s.regTokens,
		AppServices:          s.appservices,
		RateLimit:            cfg.ClientAPI.RateLimit,
		TrustXForwardedFor:   cfg.ClientAPI.TrustXForwardedFor,
		Logger:               log,
	})
	if err != nil {
		_ = s.closeDB()
		return nil, err
	}

	s.adminAPI, err = adminapi.New(adminapi.Options{
		ServerName:         cfg.Global.ServerName,
		Users:              s.users,
		Rooms:              s.rooms,
		Settings:           settingsService,
		Federation:         s.federation,
		FederationStatus:   fedStatus,
		Database:           s.db,
		Media:              s.media,
		AppServices:        s.appservices,
		Notices:            s.notices,
		RegistrationTokens: s.regTokens,
		Logger:             log,
	})
	if err != nil {
		_ = s.closeDB()
		return nil, err
	}

	// Delivery to application services starts with the server rather
	// than with Run, like the push worker: it only reads the database and
	// talks to the services, and tests that drive the handler directly
	// need it running.
	s.appservices.Start(ctx)

	s.handler = httputil.Chain(s.routes(),
		httputil.Recoverer,
		httputil.RequestLogger(log),
		s.metrics.InFlight(),
		httputil.CORS(httputil.CORSOptions{AllowedOrigins: cfg.ClientAPI.CORSAllowedOrigins}),
	)
	return s, nil
}

// prepareSchema migrates the database, or refuses to start when migrations
// are pending and auto-migration is off.
func (s *Server) prepareSchema(ctx context.Context) error {
	if s.cfg.Database.AutoMigrate {
		if err := s.db.Migrate(ctx); err != nil {
			return fmt.Errorf("migrate database: %w", err)
		}
		return nil
	}
	pending, err := s.db.PendingMigrations(ctx)
	if err != nil {
		return fmt.Errorf("check migrations: %w", err)
	}
	if len(pending) > 0 {
		return fmt.Errorf("database has %d pending migration(s): %v; run `warp migrate`", len(pending), pending)
	}
	return nil
}

// Addr reports the address Run is listening on, or "" before it starts.
// Tests and `warp serve` on port 0 use it to find the chosen port.
func (s *Server) Addr() string {
	if p := s.addr.Load(); p != nil {
		return *p
	}
	return ""
}

// Handler exposes the fully wrapped HTTP handler, for tests.
func (s *Server) Handler() http.Handler { return s.handler }

// Settings exposes the runtime-settings service; the admin API in Phase 1
// is its first user.
func (s *Server) Settings() *settings.Service { return s.settings }

// Database exposes the storage handle.
func (s *Server) Database() storage.Database { return s.db }

// Users exposes the userapi component, which `warp create-account` and the
// tests use to create accounts without going through HTTP.
func (s *Server) Users() *userapi.API { return s.users }

// Rooms exposes the roomserver component, or nil when the server started
// without a signing key.
func (s *Server) Rooms() *roomserver.API { return s.rooms }

// Sync exposes the syncapi component, or nil when the server started
// without a signing key.
func (s *Server) Sync() *syncapi.API { return s.sync }

// Keys exposes the keyserver, or nil when the server started without a
// signing key.
func (s *Server) Keys() *keyserver.API { return s.keys }

// Push exposes the push server, or nil when the server started without a
// signing key.
func (s *Server) Push() *pushserver.API { return s.push }

// AppServices exposes the application service component.
func (s *Server) AppServices() *appservice.API { return s.appservices }

// ServerNotices exposes the server notice sender, or nil when server
// notices are off or the server has no signing key.
func (s *Server) ServerNotices() *servernotices.API { return s.notices }

// Media exposes the media repository.
func (s *Server) Media() *mediaapi.API { return s.media }

// SigningKey returns the server signing key, or nil when it was skipped.
func (s *Server) SigningKey() *signing.KeyPair { return s.signingKey }

// Run listens and serves until ctx is cancelled, then shuts down
// gracefully: it stops accepting connections and gives in-flight requests
// up to listen.shutdown_timeout to finish.
//
// With listen.federation_address set there are two listeners, the plain
// one and a TLS one for other servers, serving the same handler: what a
// request may reach is decided by the gate and the routes, not by which
// port it came in on. The federation module is started here rather than
// in New, so that a server that is built and never run leaves nothing
// running behind it, and is stopped before the listeners close.
func (s *Server) Run(ctx context.Context) error {
	listener, err := net.Listen("tcp", s.cfg.Listen.Address)
	if err != nil {
		return fmt.Errorf("listen on %s: %w", s.cfg.Listen.Address, err)
	}
	listeners := []net.Listener{listener}
	if s.tlsConfig != nil {
		fedListener, err := net.Listen("tcp", s.cfg.Listen.FederationAddress)
		if err != nil {
			_ = listener.Close()
			return fmt.Errorf("listen on %s: %w", s.cfg.Listen.FederationAddress, err)
		}
		listeners = append(listeners, tls.NewListener(fedListener, s.tlsConfig))
	}

	addr := listener.Addr().String()
	s.addr.Store(&addr)
	defer s.addr.Store(nil)

	newHTTPServer := func() *http.Server {
		return &http.Server{
			Handler:           s.handler,
			ReadHeaderTimeout: s.cfg.Listen.ReadHeaderTimeout,
			ReadTimeout:       s.cfg.Listen.ReadTimeout,
			WriteTimeout:      s.cfg.Listen.WriteTimeout,
			IdleTimeout:       s.cfg.Listen.IdleTimeout,
			MaxHeaderBytes:    s.cfg.Listen.MaxHeaderBytes,
			ErrorLog:          slog.NewLogLogger(s.log.Handler(), slog.LevelDebug),
			BaseContext:       func(net.Listener) context.Context { return ctx },
		}
	}

	s.running.Store(true)
	defer s.running.Store(false)
	if s.fedModule != nil {
		if s.settings.FederationEnabled() {
			if err := s.fedModule.Start(ctx); err != nil {
				s.log.Error("could not start federation", "error", err)
			}
		}
		defer s.fedModule.Stop()
	}

	federationAddr := ""
	if len(listeners) > 1 {
		federationAddr = listeners[1].Addr().String()
		s.fedAddr.Store(&federationAddr)
		defer s.fedAddr.Store(nil)
	}
	s.log.Info("warp is listening",
		"address", addr,
		"federation_address", federationAddr,
		"server_name", s.cfg.Global.ServerName,
		"version", buildinfo.String(),
		"database", s.db.Driver(),
		"federation", s.settings.FederationEnabled(),
		"registration", s.settings.RegistrationEnabled(),
		"presence", s.cfg.Sync.Presence,
	)

	servers := make([]*http.Server, len(listeners))
	errCh := make(chan error, len(listeners))
	for i, l := range listeners {
		servers[i] = newHTTPServer()
		go func(srv *http.Server, l net.Listener) {
			err := srv.Serve(l)
			if errors.Is(err, http.ErrServerClosed) {
				err = nil
			}
			errCh <- err
		}(servers[i], l)
	}

	var serveErr error
	select {
	case serveErr = <-errCh:
		// One listener failing takes the server down: half a homeserver
		// is worse than a restart.
	case <-ctx.Done():
	}

	s.log.Info("shutting down", "timeout", s.cfg.Listen.ShutdownTimeout)
	if s.fedModule != nil {
		s.fedModule.Stop()
	}
	shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.cfg.Listen.ShutdownTimeout)
	defer cancel()

	for _, srv := range servers {
		if err := srv.Shutdown(shutdownCtx); err != nil {
			s.log.Warn("graceful shutdown timed out; closing connections", "error", err)
			_ = srv.Close()
		}
	}
	// Every Serve has returned by now; surface anything they reported.
	remaining := len(servers)
	if serveErr != nil {
		remaining--
	}
	for range remaining {
		if err := <-errCh; err != nil && serveErr == nil {
			serveErr = err
		}
	}
	if serveErr != nil {
		return serveErr
	}
	s.log.Info("stopped")
	return nil
}

// Close releases the resources New acquired.
func (s *Server) Close() error {
	if s.fedModule != nil {
		s.fedModule.Stop()
	}
	if s.keys != nil {
		// Background fetches of remote keys write to the database, which
		// is about to close.
		s.keys.Close()
	}
	if s.sync != nil {
		// The typing tracker holds a timer per typing user; a server
		// shutting down should not wait for them.
		s.sync.Close()
	}
	if s.push != nil {
		// The push worker writes notifications, and the database is about
		// to close.
		s.push.Close()
	}
	if s.appservices != nil {
		// So do the application service workers, which move their
		// cursors.
		s.appservices.Close()
	}
	if s.stopTracing != nil {
		// Spans still in the batch are sent rather than lost.
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		if err := s.stopTracing(ctx); err != nil {
			s.log.Warn("could not flush traces", "error", err)
		}
	}
	return s.closeDB()
}

func (s *Server) closeDB() error {
	if s.db == nil || !s.ownsDB {
		return nil
	}
	err := s.db.Close()
	s.db = nil
	return err
}

// contextWithTimeout is a thin wrapper so handlers do not import context
// just for a deadline.
func contextWithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) {
	return context.WithTimeout(ctx, d)
}

// FederationModule exposes the federation module, or nil when the server
// started without a signing key. Tests that never call Run start it
// themselves.
func (s *Server) FederationModule() *federationapi.Module { return s.fedModule }

// FederationAddr reports the address of the TLS federation listener while
// Run is serving one, or "".
func (s *Server) FederationAddr() string {
	if p := s.fedAddr.Load(); p != nil {
		return *p
	}
	return ""
}