Phase 5: federation

Implements PLAN.md Phase 5: the Server-Server API. Warp joins rooms on
other servers and lets their users join its rooms, invites across servers
with the invitee's server countersigning, rejects remote invites (locally
if the inviter is unreachable), and exchanges events, typing, receipts and
to-device messages with other homeservers — all behind the runtime switch,
which now starts and stops real outbound traffic.

The protocol lives in internal/federation with no knowledge of rooms:
X-Matrix signing (fuzzed parser), the spec's discovery algorithm keeping
the dial address and the TLS name apart, a signed client that refuses
every request while the switch is off or the policy denies the
destination, and a key ring that caches other servers' keys for at most a
week, with optional notaries.

The roomserver takes events from elsewhere through the spec's intake:
signatures and hashes (a hash mismatch keeps the redacted form), the rules
against the event's own auth events, against the state before it — with
/get_missing_events for small gaps and /state_ids for wide ones — and
against the current state, where a failure is a soft failure. Events held
without their state are outliers: a column on roomserver_events, because
the current state of a room joined over federation is almost all
outliers, and excluded from every timeline query. Forks are resolved at
the room's head. Membership across servers is make_join/send_join,
make_leave/send_leave and countersigned invites, in both directions;
m.room.server_acl is enforced on everything another server sends.
History before a remote join is backfilled when /messages pages past it,
stored at negative stream positions.

internal/federationapi is the running component: the inbound endpoints,
and an outbound queue that persists what it owes each destination, sends
it in transactions of 50 PDUs and 100 EDUs, backs off from ten seconds to
six hours, survives restarts, and drops events persisted while federation
is off rather than delivering a backlog in one burst. Its Module is what
the switch starts and stops.

Also: a TLS listener for port 8448, trusted key servers, key validity and
an outbound CA bundle in the config; per-destination health and backoff
reset in the admin API; remote profiles, aliases and directories in the
Client-Server API; a Complement federation profile. See
docs/federation.md, including its known gaps (no partial-state joins, no
key rotation).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/storage/sqlstore/federation.go

@@ -0,0 +1,323 @@
+package sqlstore
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/dariuszjaniak/warp/server/internal/storage"
+)
+
+// Federation is the shared SQL implementation of storage.FederationStore.
+type Federation struct {
+	db     *sql.DB
+	rebind func(string) string
+}
+
+// NewFederation builds a federation store.
+func NewFederation(db *sql.DB, rebind func(string) string) *Federation {
+	if rebind == nil {
+		rebind = func(query string) string { return query }
+	}
+	return &Federation{db: db, rebind: rebind}
+}
+
+// The sequence table of this component, and its one counter.
+const (
+	federationSequences = "federation_sequences"
+	seqQueue            = "queue"
+)
+
+func (s *Federation) q(query string) string { return s.rebind(query) }
+
+func (s *Federation) tx(ctx context.Context, fn func(*sql.Tx) error) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin transaction: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+	if err := fn(tx); err != nil {
+		return err
+	}
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit transaction: %w", err)
+	}
+	return nil
+}
+
+const upsertServerKeySQL = `
+	INSERT INTO federation_server_keys
+		(server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms)
+	VALUES (?, ?, ?, ?, ?, ?)
+	ON CONFLICT (server_name, key_id) DO UPDATE SET
+		public_key = excluded.public_key,
+		valid_until_ts = excluded.valid_until_ts,
+		expired_ts = excluded.expired_ts,
+		fetched_at_ms = excluded.fetched_at_ms`
+
+// StoreServerKeys implements storage.FederationStore.
+func (s *Federation) StoreServerKeys(ctx context.Context, keys []storage.ServerKey) error {
+	if len(keys) == 0 {
+		return nil
+	}
+	return s.tx(ctx, func(tx *sql.Tx) error {
+		for _, k := range keys {
+			if _, err := tx.ExecContext(ctx, s.q(upsertServerKeySQL),
+				k.ServerName, k.KeyID, k.PublicKey, k.ValidUntilTS, k.ExpiredTS, k.FetchedAtMS); err != nil {
+				return fmt.Errorf("store key %s of %s: %w", k.KeyID, k.ServerName, err)
+			}
+		}
+		return nil
+	})
+}
+
+// ServerKeys implements storage.FederationStore.
+func (s *Federation) ServerKeys(ctx context.Context, serverName string, keyIDs []string) ([]storage.ServerKey, error) {
+	query := `
+		SELECT server_name, key_id, public_key, valid_until_ts, expired_ts, fetched_at_ms
+		FROM federation_server_keys WHERE server_name = ?`
+	args := []any{serverName}
+	if len(keyIDs) > 0 {
+		query += ` AND key_id IN (` + placeholders(len(keyIDs)) + `)`
+		for _, id := range keyIDs {
+			args = append(args, id)
+		}
+	}
+	query += ` ORDER BY key_id`
+
+	rows, err := s.db.QueryContext(ctx, s.q(query), args...)
+	if err != nil {
+		return nil, fmt.Errorf("read the keys of %q: %w", serverName, err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []storage.ServerKey
+	for rows.Next() {
+		var k storage.ServerKey
+		if err := rows.Scan(&k.ServerName, &k.KeyID, &k.PublicKey,
+			&k.ValidUntilTS, &k.ExpiredTS, &k.FetchedAtMS); err != nil {
+			return nil, fmt.Errorf("read the keys of %q: %w", serverName, err)
+		}
+		out = append(out, k)
+	}
+	return out, rows.Err()
+}
+
+const insertQueuedSQL = `
+	INSERT INTO federation_queue
+		(queue_id, destination, kind, room_id, event_id, payload, queued_at_ms)
+	VALUES (?, ?, ?, ?, ?, ?, ?)`
+
+// Enqueue implements storage.FederationStore.
+func (s *Federation) Enqueue(ctx context.Context, items []storage.QueuedItem) ([]storage.QueuedItem, error) {
+	if len(items) == 0 {
+		return nil, nil
+	}
+	out := make([]storage.QueuedItem, len(items))
+	copy(out, items)
+	err := s.tx(ctx, func(tx *sql.Tx) error {
+		top, err := advanceSequence(ctx, tx, s.q, federationSequences, seqQueue, int64(len(out)))
+		if err != nil {
+			return err
+		}
+		first := top - int64(len(out)) + 1
+		for i := range out {
+			out[i].ID = first + int64(i)
+			item := out[i]
+			if _, err := tx.ExecContext(ctx, s.q(insertQueuedSQL),
+				item.ID, item.Destination, item.Kind, item.RoomID, item.EventID,
+				string(item.Payload), item.QueuedAtMS); err != nil {
+				return fmt.Errorf("queue for %s: %w", item.Destination, err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("enqueue %d items: %w", len(items), err)
+	}
+	return out, nil
+}
+
+const selectQueuedSQL = `
+	SELECT queue_id, destination, kind, room_id, event_id, payload, queued_at_ms
+	FROM federation_queue
+	WHERE destination = ? AND kind = ?
+	ORDER BY queue_id
+	LIMIT ?`
+
+// Queued implements storage.FederationStore.
+func (s *Federation) Queued(ctx context.Context, destination, kind string, limit int) ([]storage.QueuedItem, error) {
+	if limit <= 0 {
+		return nil, nil
+	}
+	rows, err := s.db.QueryContext(ctx, s.q(selectQueuedSQL), destination, kind, limit)
+	if err != nil {
+		return nil, fmt.Errorf("read the queue for %q: %w", destination, err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []storage.QueuedItem
+	for rows.Next() {
+		var (
+			item    storage.QueuedItem
+			payload string
+		)
+		if err := rows.Scan(&item.ID, &item.Destination, &item.Kind, &item.RoomID,
+			&item.EventID, &payload, &item.QueuedAtMS); err != nil {
+			return nil, fmt.Errorf("read the queue for %q: %w", destination, err)
+		}
+		item.Payload = []byte(payload)
+		out = append(out, item)
+	}
+	return out, rows.Err()
+}
+
+// Dequeue implements storage.FederationStore.
+func (s *Federation) Dequeue(ctx context.Context, ids []int64) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	args := make([]any, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+	query := `DELETE FROM federation_queue WHERE queue_id IN (` + placeholders(len(ids)) + `)`
+	if _, err := s.db.ExecContext(ctx, s.q(query), args...); err != nil {
+		return fmt.Errorf("dequeue %d items: %w", len(ids), err)
+	}
+	return nil
+}
+
+const selectQueuedDestinationsSQL = `
+	SELECT DISTINCT destination FROM federation_queue ORDER BY destination`
+
+// QueuedDestinations implements storage.FederationStore.
+func (s *Federation) QueuedDestinations(ctx context.Context) ([]string, error) {
+	rows, err := s.db.QueryContext(ctx, s.q(selectQueuedDestinationsSQL))
+	if err != nil {
+		return nil, fmt.Errorf("read the queued destinations: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []string
+	for rows.Next() {
+		var d string
+		if err := rows.Scan(&d); err != nil {
+			return nil, fmt.Errorf("read the queued destinations: %w", err)
+		}
+		out = append(out, d)
+	}
+	return out, rows.Err()
+}
+
+const destinationColumns = `
+	server_name, failures, retry_at_ms, last_success_ms, last_failure_ms, last_error`
+
+func scanDestination(scan func(...any) error) (storage.Destination, error) {
+	var d storage.Destination
+	err := scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS, &d.LastFailureMS, &d.LastError)
+	return d, err
+}
+
+const selectDestinationSQL = `SELECT ` + destinationColumns + `
+	FROM federation_destinations WHERE server_name = ?`
+
+// Destination implements storage.FederationStore.
+func (s *Federation) Destination(ctx context.Context, serverName string) (storage.Destination, error) {
+	d, err := scanDestination(s.db.QueryRowContext(ctx, s.q(selectDestinationSQL), serverName).Scan)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return storage.Destination{ServerName: serverName}, nil
+	case err != nil:
+		return storage.Destination{}, fmt.Errorf("read destination %q: %w", serverName, err)
+	}
+	return d, nil
+}
+
+const upsertDestinationSQL = `
+	INSERT INTO federation_destinations (` + destinationColumns + `)
+	VALUES (?, ?, ?, ?, ?, ?)
+	ON CONFLICT (server_name) DO UPDATE SET
+		failures = excluded.failures,
+		retry_at_ms = excluded.retry_at_ms,
+		last_success_ms = excluded.last_success_ms,
+		last_failure_ms = excluded.last_failure_ms,
+		last_error = excluded.last_error`
+
+// SetDestination implements storage.FederationStore.
+func (s *Federation) SetDestination(ctx context.Context, d storage.Destination) error {
+	if _, err := s.db.ExecContext(ctx, s.q(upsertDestinationSQL),
+		d.ServerName, d.Failures, d.RetryAtMS, d.LastSuccessMS, d.LastFailureMS, d.LastError); err != nil {
+		return fmt.Errorf("record destination %q: %w", d.ServerName, err)
+	}
+	return nil
+}
+
+// selectDestinationSummariesSQL lists every server that either has a
+// health row or something queued. The union is what makes a destination
+// that has never failed, but has a backlog, show up at all.
+const selectDestinationSummariesSQL = `
+	SELECT n.server_name,
+		COALESCE(d.failures, 0), COALESCE(d.retry_at_ms, 0),
+		COALESCE(d.last_success_ms, 0), COALESCE(d.last_failure_ms, 0),
+		COALESCE(d.last_error, ''),
+		(SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'pdu'),
+		(SELECT COUNT(*) FROM federation_queue q WHERE q.destination = n.server_name AND q.kind = 'edu')
+	FROM (
+		SELECT server_name FROM federation_destinations
+		UNION
+		SELECT DISTINCT destination FROM federation_queue
+	) n
+	LEFT JOIN federation_destinations d ON d.server_name = n.server_name
+	ORDER BY n.server_name`
+
+// Destinations implements storage.FederationStore.
+func (s *Federation) Destinations(ctx context.Context) ([]storage.DestinationSummary, error) {
+	rows, err := s.db.QueryContext(ctx, s.q(selectDestinationSummariesSQL))
+	if err != nil {
+		return nil, fmt.Errorf("read the destinations: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []storage.DestinationSummary
+	for rows.Next() {
+		var d storage.DestinationSummary
+		if err := rows.Scan(&d.ServerName, &d.Failures, &d.RetryAtMS, &d.LastSuccessMS,
+			&d.LastFailureMS, &d.LastError, &d.QueuedPDUs, &d.QueuedEDUs); err != nil {
+			return nil, fmt.Errorf("read the destinations: %w", err)
+		}
+		out = append(out, d)
+	}
+	return out, rows.Err()
+}
+
+const selectInboundTxnSQL = `
+	SELECT response_json FROM federation_inbound_txns WHERE origin = ? AND txn_id = ?`
+
+// InboundTransaction implements storage.FederationStore.
+func (s *Federation) InboundTransaction(ctx context.Context, origin, txnID string) ([]byte, bool, error) {
+	var response string
+	err := s.db.QueryRowContext(ctx, s.q(selectInboundTxnSQL), origin, txnID).Scan(&response)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return nil, false, nil
+	case err != nil:
+		return nil, false, fmt.Errorf("read transaction %q from %q: %w", txnID, origin, err)
+	}
+	return []byte(response), true, nil
+}
+
+const insertInboundTxnSQL = `
+	INSERT INTO federation_inbound_txns (origin, txn_id, response_json, received_at_ms)
+	VALUES (?, ?, ?, ?)
+	ON CONFLICT (origin, txn_id) DO NOTHING`
+
+// RecordInboundTransaction implements storage.FederationStore.
+func (s *Federation) RecordInboundTransaction(ctx context.Context, origin, txnID string, response []byte, atMS int64) error {
+	if _, err := s.db.ExecContext(ctx, s.q(insertInboundTxnSQL),
+		origin, txnID, string(response), atMS); err != nil {
+		return fmt.Errorf("record transaction %q from %q: %w", txnID, origin, err)
+	}
+	return nil
+}

server/internal/storage/sqlstore/rooms.go

@@ -46,10 +46,14 @@ const (
 	maxRoomPageSize     = 1000
 )
 
-// The two counters in roomserver_sequences.
+// The counters in roomserver_sequences.
 const (
 	seqStreamOrdering = "stream_ordering"
 	seqStateSnapshot  = "state_snapshot"
+
+	// seqBackfillOrdering numbers backfilled history. It starts at 0 and
+	// only ever goes down; see WriteBackfill.
+	seqBackfillOrdering = "backfill_ordering"
 )
 
 // q rebinds a statement for this driver.
@@ -121,14 +125,22 @@ func (s *Rooms) Write(ctx context.Context, w storage.RoomWrite) (int64, error) {
 			return err
 		}
 
-		snapshotID, err := s.stateSnapshot(ctx, tx, w.Event.RoomID, w.StateBefore)
-		if err != nil {
-			return err
+		// An outlier has no state snapshot: that is what makes it one.
+		var snapshotID int64
+		if !w.Outlier {
+			if snapshotID, err = s.stateSnapshot(ctx, tx, w.Event.RoomID, w.StateBefore); err != nil {
+				return err
+			}
 		}
 
-		if err := s.insertEvent(ctx, tx, w.Event, ordering, snapshotID); err != nil {
+		if err := s.insertEvent(ctx, tx, w.Event, ordering, snapshotID, w.Outlier); err != nil {
 			return err
 		}
+		if w.Event.Type == memberEventType && w.Event.StateKey != nil {
+			if err := s.replaceInviteState(ctx, tx, w.Event.RoomID, *w.Event.StateKey, w.InviteState); err != nil {
+				return err
+			}
+		}
 		if w.Redacts != "" {
 			if err := s.applyRedaction(ctx, tx, w.Event.RoomID, w.Redacts,
 				w.Event.EventID, w.RedactedJSON); err != nil {
@@ -246,21 +258,55 @@ func stateHash(entries []storage.StateEntry) string {
 	return base64.RawURLEncoding.EncodeToString(h.Sum(nil))
 }
 
-const insertEventSQL = `
-	INSERT INTO roomserver_events (
-		event_id, room_id, stream_ordering, event_type, state_key,
-		sender, depth, origin_server_ts, event_json, state_before_id)
-	VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+// memberEventType is the one event type the store has to recognise: its
+// rows carry a membership, and its arrival clears a stored invite.
+const memberEventType = "m.room.member"
 
-func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64) error {
-	_, err := tx.ExecContext(ctx, s.q(insertEventSQL),
+const (
+	insertEventSQL = `
+		INSERT INTO roomserver_events (
+			event_id, room_id, stream_ordering, event_type, state_key,
+			sender, depth, origin_server_ts, event_json, state_before_id, outlier)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
+	selectOutlierSQL  = `SELECT outlier FROM roomserver_events WHERE event_id = ?`
+	promoteOutlierSQL = `
+		UPDATE roomserver_events
+		SET stream_ordering = ?, state_before_id = ?, outlier = ?
+		WHERE event_id = ? AND outlier = ?`
+)
+
+// insertEvent writes the event row.
+//
+// An event this server already holds is not written twice. The one
+// exception is an outlier arriving again as an ordinary event, which is
+// promoted in place: it gets the new stream position and the state
+// snapshot it now has, and joins the timeline. Its JSON is left as it was —
+// the event ID says the two copies are the same event, and the stored one
+// may already have been redacted.
+func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEvent, ordering, snapshotID int64, outlier bool) error {
+	var existingOutlier bool
+	err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), e.EventID).Scan(&existingOutlier)
+	switch {
+	case err == nil:
+		if existingOutlier && !outlier {
+			if _, err := tx.ExecContext(ctx, s.q(promoteOutlierSQL),
+				ordering, snapshotID, false, e.EventID, true); err != nil {
+				return fmt.Errorf("promote outlier: %w", err)
+			}
+		}
+		// The same event twice is not a failure worth propagating: an
+		// event ID is a hash of its contents, so the second copy says
+		// exactly what the first one did.
+		return nil
+	case !errors.Is(err, sql.ErrNoRows):
+		return fmt.Errorf("look up event: %w", err)
+	}
+
+	_, err = tx.ExecContext(ctx, s.q(insertEventSQL),
 		e.EventID, e.RoomID, ordering, e.Type, e.StateKey,
-		e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID)
+		e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, outlier)
 	if err != nil {
 		if s.isUnique(err) {
-			// The same event twice is not a failure worth propagating: an
-			// event ID is a hash of its contents, so the second copy says
-			// exactly what the first one did.
 			return nil
 		}
 		return fmt.Errorf("insert event: %w", err)
@@ -268,6 +314,147 @@ func (s *Rooms) insertEvent(ctx context.Context, tx *sql.Tx, e storage.StoredEve
 	return nil
 }
 
+const (
+	deleteInviteStateSQL = `DELETE FROM roomserver_invite_state WHERE room_id = ? AND user_id = ?`
+	insertInviteStateSQL = `
+		INSERT INTO roomserver_invite_state (room_id, user_id, state_json) VALUES (?, ?, ?)`
+)
+
+// replaceInviteState clears a user's stored invite state and, for an
+// invite that came with some, stores the new one.
+func (s *Rooms) replaceInviteState(ctx context.Context, tx *sql.Tx, roomID, userID string, state []byte) error {
+	if _, err := tx.ExecContext(ctx, s.q(deleteInviteStateSQL), roomID, userID); err != nil {
+		return fmt.Errorf("clear invite state: %w", err)
+	}
+	if len(state) == 0 {
+		return nil
+	}
+	if _, err := tx.ExecContext(ctx, s.q(insertInviteStateSQL), roomID, userID, string(state)); err != nil {
+		return fmt.Errorf("store invite state: %w", err)
+	}
+	return nil
+}
+
+const selectInviteStateSQL = `
+	SELECT state_json FROM roomserver_invite_state WHERE room_id = ? AND user_id = ?`
+
+// InviteState implements storage.RoomStore.
+func (s *Rooms) InviteState(ctx context.Context, roomID, userID string) ([]byte, error) {
+	var state string
+	err := s.db.QueryRowContext(ctx, s.q(selectInviteStateSQL), roomID, userID).Scan(&state)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return nil, nil
+	case err != nil:
+		return nil, fmt.Errorf("read the invite state of %q in %q: %w", userID, roomID, err)
+	}
+	return []byte(state), nil
+}
+
+// WriteOutliers implements storage.RoomStore.
+//
+// The stream positions are taken as one block, in one UPDATE, rather than
+// one at a time: a join to a large room hands over tens of thousands of
+// events, and a counter bumped per event would be that many statements
+// holding the counter's row lock.
+func (s *Rooms) WriteOutliers(ctx context.Context, roomID string, events []storage.StoredEvent) error {
+	if len(events) == 0 {
+		return nil
+	}
+	err := s.tx(ctx, func(tx *sql.Tx) error {
+		top, err := advanceSequence(ctx, tx, s.q, roomSequences, seqStreamOrdering, int64(len(events)))
+		if err != nil {
+			return err
+		}
+		first := top - int64(len(events)) + 1
+		for i, e := range events {
+			if e.RoomID == "" {
+				e.RoomID = roomID
+			}
+			if err := s.insertEvent(ctx, tx, e, first+int64(i), 0, true); err != nil {
+				return fmt.Errorf("outlier %s: %w", e.EventID, err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("write %d outliers to %q: %w", len(events), roomID, err)
+	}
+	return nil
+}
+
+// WriteBackfill implements storage.RoomStore.
+//
+// The positions come from their own counter, advanced downwards by the
+// number of events actually written, in one UPDATE: the events already in
+// the timeline are sorted out first, inside the transaction, so that the
+// block taken is exactly as big as what goes into it. A hole below the
+// room would be harmless — nothing streams through backfilled history, it
+// is only ever read by bounds — but there is no reason to leave one.
+//
+// The batch takes the block oldest first: if the counter stood at -10 and
+// three events are written, they become -13, -12 and -11.
+func (s *Rooms) WriteBackfill(ctx context.Context, roomID string, events []storage.BackfillEvent) error {
+	if len(events) == 0 {
+		return nil
+	}
+	err := s.tx(ctx, func(tx *sql.Tx) error {
+		// Which of the batch is new, which is an outlier to promote, and
+		// which is already in the timeline and left alone.
+		var write []storage.BackfillEvent
+		promote := map[string]bool{}
+		for _, b := range events {
+			var outlier bool
+			err := tx.QueryRowContext(ctx, s.q(selectOutlierSQL), b.Event.EventID).Scan(&outlier)
+			switch {
+			case errors.Is(err, sql.ErrNoRows):
+				write = append(write, b)
+			case err != nil:
+				return fmt.Errorf("look up event %s: %w", b.Event.EventID, err)
+			case outlier:
+				write = append(write, b)
+				promote[b.Event.EventID] = true
+			}
+		}
+		if len(write) == 0 {
+			return nil
+		}
+
+		bottom, err := advanceSequence(ctx, tx, s.q, roomSequences, seqBackfillOrdering, -int64(len(write)))
+		if err != nil {
+			return err
+		}
+		for i, b := range write {
+			e := b.Event
+			if e.RoomID == "" {
+				e.RoomID = roomID
+			}
+			ordering := bottom + int64(i)
+			snapshotID, err := s.stateSnapshot(ctx, tx, e.RoomID, b.StateBefore)
+			if err != nil {
+				return err
+			}
+			if promote[e.EventID] {
+				if _, err := tx.ExecContext(ctx, s.q(promoteOutlierSQL),
+					ordering, snapshotID, false, e.EventID, true); err != nil {
+					return fmt.Errorf("promote outlier %s: %w", e.EventID, err)
+				}
+				continue
+			}
+			if _, err := tx.ExecContext(ctx, s.q(insertEventSQL),
+				e.EventID, e.RoomID, ordering, e.Type, e.StateKey,
+				e.Sender, e.Depth, e.OriginServerTS, string(e.JSON), snapshotID, false); err != nil {
+				return fmt.Errorf("insert event %s: %w", e.EventID, err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("backfill %d events into %q: %w", len(events), roomID, err)
+	}
+	return nil
+}
+
 const applyRedactionSQL = `
 	UPDATE roomserver_events SET event_json = ?, redacted_by = ?
 	WHERE room_id = ? AND event_id = ? AND redacted_by = ''`
@@ -387,7 +574,7 @@ func (s *Rooms) recordTransaction(ctx context.Context, tx *sql.Tx, ref storage.T
 // scanEvent expects.
 const eventColumns = `
 	e.room_id, e.event_id, e.event_type, e.state_key, e.sender,
-	e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by`
+	e.depth, e.origin_server_ts, e.event_json, e.stream_ordering, e.redacted_by, e.outlier`
 
 // scanEvent reads one row of eventColumns.
 func scanEvent(scan func(...any) error) (storage.StoredEvent, error) {
@@ -397,7 +584,7 @@ func scanEvent(scan func(...any) error) (storage.StoredEvent, error) {
 		body     string
 	)
 	err := scan(&e.RoomID, &e.EventID, &e.Type, &stateKey, &e.Sender,
-		&e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy)
+		&e.Depth, &e.OriginServerTS, &body, &e.StreamOrdering, &e.RedactedBy, &e.Outlier)
 	if err != nil {
 		return storage.StoredEvent{}, err
 	}
@@ -892,6 +1079,7 @@ var purgeStatements = []string{
 	`DELETE FROM roomserver_extremities WHERE room_id = ?`,
 	`DELETE FROM roomserver_transactions WHERE room_id = ?`,
 	`DELETE FROM roomserver_aliases WHERE room_id = ?`,
+	`DELETE FROM roomserver_invite_state WHERE room_id = ?`,
 	`DELETE FROM roomserver_events WHERE room_id = ?`,
 	`DELETE FROM roomserver_rooms WHERE room_id = ?`,
 }
@@ -968,3 +1156,42 @@ func expectOneRow(result sql.Result, subject string, notFound error) error {
 	}
 	return nil
 }
+
+var selectEventByIDSQL = `
+	SELECT ` + eventColumns + `
+	FROM roomserver_events e WHERE e.event_id = ?`
+
+// EventByID implements storage.RoomStore.
+func (s *Rooms) EventByID(ctx context.Context, eventID string) (storage.StoredEvent, error) {
+	e, err := scanEvent(s.db.QueryRowContext(ctx, s.q(selectEventByIDSQL), eventID).Scan)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return storage.StoredEvent{}, fmt.Errorf("event %q: %w", eventID, storage.ErrNoSuchEvent)
+	case err != nil:
+		return storage.StoredEvent{}, fmt.Errorf("read event %q: %w", eventID, err)
+	}
+	return e, nil
+}
+
+const selectJoinedMemberOnSQL = `
+	SELECT 1 FROM roomserver_current_state
+	WHERE room_id = ? AND event_type = 'm.room.member' AND membership = 'join'
+		AND state_key LIKE ? ESCAPE '\'
+	LIMIT 1`
+
+// HasJoinedMemberOn implements storage.RoomStore.
+//
+// A user ID's server is everything after its first colon, and a localpart
+// cannot contain one, so "ends with ':' plus the server name" is exactly
+// "belongs to that server".
+func (s *Rooms) HasJoinedMemberOn(ctx context.Context, roomID, serverName string) (bool, error) {
+	var one int
+	err := s.db.QueryRowContext(ctx, s.q(selectJoinedMemberOnSQL), roomID, `%:`+escapeLike(serverName)).Scan(&one)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return false, nil
+	case err != nil:
+		return false, fmt.Errorf("look for members of %q in %q: %w", serverName, roomID, err)
+	}
+	return true, nil
+}

server/internal/storage/sqlstore/sequences.go

@@ -44,8 +44,15 @@ type execQuerier interface {
 // statements are two transactions, and two callers can then be handed the
 // same value.
 func nextSequence(ctx context.Context, db execQuerier, q func(string) string, table, name string) (int64, error) {
-	bump := fmt.Sprintf(`UPDATE %s SET value = value + 1 WHERE name = ?`, table)
-	if _, err := db.ExecContext(ctx, q(bump), name); err != nil {
+	return advanceSequence(ctx, db, q, table, name, 1)
+}
+
+// advanceSequence hands out a block of n values and returns the last of
+// them; the block is the n values ending there. Like nextSequence it must
+// be called inside a transaction.
+func advanceSequence(ctx context.Context, db execQuerier, q func(string) string, table, name string, n int64) (int64, error) {
+	bump := fmt.Sprintf(`UPDATE %s SET value = value + ? WHERE name = ?`, table)
+	if _, err := db.ExecContext(ctx, q(bump), n, name); err != nil {
 		return 0, fmt.Errorf("advance the %s counter: %w", name, err)
 	}
 	return currentSequence(ctx, db, q, table, name)

server/internal/storage/sqlstore/sqlstore.go

@@ -59,6 +59,7 @@ type Store struct {
 	users    *Users
 	rooms    *Rooms
 	sync     *Sync
+	fed      *Federation
 }
 
 // New builds a Store. It does not touch the database.
@@ -77,6 +78,7 @@ func New(opts Options) (*Store, error) {
 		users:    NewUsers(opts.DB, opts.Rebind, opts.IsUniqueViolation),
 		rooms:    NewRooms(opts.DB, opts.Rebind, opts.IsUniqueViolation),
 		sync:     NewSync(opts.DB, opts.Rebind, opts.IsUniqueViolation),
+		fed:      NewFederation(opts.DB, opts.Rebind),
 	}, nil
 }
 
@@ -104,6 +106,9 @@ func (s *Store) Rooms() storage.RoomStore { return s.rooms }
 // Sync implements storage.Database.
 func (s *Store) Sync() storage.SyncStore { return s.sync }
 
+// Federation implements storage.Database.
+func (s *Store) Federation() storage.FederationStore { return s.fed }
+
 // Migrate implements storage.Database.
 func (s *Store) Migrate(ctx context.Context) error {
 	ms, err := s.migrations()

server/internal/storage/sqlstore/timeline.go

@@ -75,7 +75,7 @@ func (s *Rooms) MembershipsForUser(ctx context.Context, userID string, membershi
 var selectUserMembershipEventsSQL = `
 	SELECT ` + eventColumns + `
 	FROM roomserver_events e
-	WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.state_key = ?
+	WHERE e.room_id = ? AND e.event_type = 'm.room.member' AND e.state_key = ? AND NOT e.outlier
 	ORDER BY e.stream_ordering`
 
 // UserMembershipEvents implements storage.RoomStore.
@@ -97,13 +97,17 @@ func (s *Rooms) Timeline(ctx context.Context, req storage.TimelineRequest) ([]st
 		return nil, nil
 	}
 
-	conditions := []string{`e.room_id = ?`}
+	// Outliers have no place in a timeline; see the outlier column.
+	conditions := []string{`e.room_id = ?`, `NOT e.outlier`}
 	args := []any{req.RoomID}
-	if req.Low > 0 {
+	// Zero is "unbounded", not a position, at either end: a bound can be
+	// negative, inside backfilled history, so "is it set" cannot be
+	// spelled "is it positive".
+	if req.Low != 0 {
 		conditions = append(conditions, `e.stream_ordering > ?`)
 		args = append(args, req.Low)
 	}
-	if req.High > 0 {
+	if req.High != 0 {
 		conditions = append(conditions, `e.stream_ordering <= ?`)
 		args = append(args, req.High)
 	}
@@ -229,13 +233,16 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to
 
 	query := `
 		SELECT DISTINCT room_id FROM roomserver_events
-		WHERE room_id IN (` + placeholders(len(roomIDs)) + `) AND stream_ordering > ?`
+		WHERE room_id IN (` + placeholders(len(roomIDs)) + `) AND NOT outlier`
 	args := make([]any, 0, len(roomIDs)+2)
 	for _, id := range roomIDs {
 		args = append(args, id)
 	}
-	args = append(args, from)
-	if to > 0 {
+	if from != 0 {
+		query += ` AND stream_ordering > ?`
+		args = append(args, from)
+	}
+	if to != 0 {
 		query += ` AND stream_ordering <= ?`
 		args = append(args, to)
 	}
@@ -261,7 +268,7 @@ func (s *Rooms) RoomsWithEvents(ctx context.Context, roomIDs []string, from, to
 const selectEventAtStreamSQL = `
 	SELECT e.event_id, e.event_type, e.state_key, e.state_before_id
 	FROM roomserver_events e
-	WHERE e.room_id = ? AND e.stream_ordering <= ?
+	WHERE e.room_id = ? AND NOT e.outlier AND e.stream_ordering <= ?
 	ORDER BY e.stream_ordering DESC
 	LIMIT 1`
 
@@ -278,7 +285,7 @@ const selectSnapshotEntriesSQL = `
 // the snapshot is the state the event was authorised against, which is the
 // state *before* it.
 func (s *Rooms) StateAtStream(ctx context.Context, roomID string, pos int64) ([]storage.StateEntry, error) {
-	if pos <= 0 {
+	if pos == 0 {
 		return nil, nil
 	}