Phase 5: federation

Implements PLAN.md Phase 5: the Server-Server API. Warp joins rooms on
other servers and lets their users join its rooms, invites across servers
with the invitee's server countersigning, rejects remote invites (locally
if the inviter is unreachable), and exchanges events, typing, receipts and
to-device messages with other homeservers — all behind the runtime switch,
which now starts and stops real outbound traffic.

The protocol lives in internal/federation with no knowledge of rooms:
X-Matrix signing (fuzzed parser), the spec's discovery algorithm keeping
the dial address and the TLS name apart, a signed client that refuses
every request while the switch is off or the policy denies the
destination, and a key ring that caches other servers' keys for at most a
week, with optional notaries.

The roomserver takes events from elsewhere through the spec's intake:
signatures and hashes (a hash mismatch keeps the redacted form), the rules
against the event's own auth events, against the state before it — with
/get_missing_events for small gaps and /state_ids for wide ones — and
against the current state, where a failure is a soft failure. Events held
without their state are outliers: a column on roomserver_events, because
the current state of a room joined over federation is almost all
outliers, and excluded from every timeline query. Forks are resolved at
the room's head. Membership across servers is make_join/send_join,
make_leave/send_leave and countersigned invites, in both directions;
m.room.server_acl is enforced on everything another server sends.
History before a remote join is backfilled when /messages pages past it,
stored at negative stream positions.

internal/federationapi is the running component: the inbound endpoints,
and an outbound queue that persists what it owes each destination, sends
it in transactions of 50 PDUs and 100 EDUs, backs off from ten seconds to
six hours, survives restarts, and drops events persisted while federation
is off rather than delivering a backlog in one burst. Its Module is what
the switch starts and stops.

Also: a TLS listener for port 8448, trusted key servers, key validity and
an outbound CA bundle in the config; per-destination health and backoff
reset in the admin API; remote profiles, aliases and directories in the
Client-Server API; a Complement federation profile. See
docs/federation.md, including its known gaps (no partial-state joins, no
key rotation).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

server/internal/clientapi/clientapi.go

@@ -36,6 +36,7 @@ type ClientAPI struct {
 	sync          *syncapi.API
 	settings      *settings.Service
 	federation    *federation.Gate
+	remoteProfile RemoteProfiles
 	registration  config.Registration
 	log           *slog.Logger
 	now           func() time.Time
@@ -91,6 +92,10 @@ type Options struct {
 	// Federation gates the endpoints that would need a remote server.
 	Federation *federation.Gate
 
+	// RemoteProfiles asks another server for one of its users' profile.
+	// Nil means remote profiles cannot be looked up at all.
+	RemoteProfiles RemoteProfiles
+
 	// Registration is the file configuration of the registration flow.
 	Registration config.Registration
 
@@ -135,6 +140,7 @@ func New(opts Options) (*ClientAPI, error) {
 		sync:          opts.Sync,
 		settings:      opts.Settings,
 		federation:    opts.Federation,
+		remoteProfile: opts.RemoteProfiles,
 		registration:  opts.Registration,
 		log:           opts.Logger,
 		now:           opts.Now,

server/internal/clientapi/profile.go

@@ -1,11 +1,13 @@
 package clientapi
 
 import (
+	"context"
 	"errors"
 	"net/http"
 
 	"github.com/dariuszjaniak/warp/server/internal/federation"
 	"github.com/dariuszjaniak/warp/server/internal/httputil"
+	"github.com/dariuszjaniak/warp/server/internal/matrix/id"
 	"github.com/dariuszjaniak/warp/server/internal/storage"
 	"github.com/dariuszjaniak/warp/server/internal/userapi"
 )
@@ -59,24 +61,29 @@ func (c *ClientAPI) handleGetAvatarURL(w http.ResponseWriter, r *http.Request) e
 	return nil
 }
 
-// lookupProfile resolves the user ID in the path to a local account.
+// RemoteProfiles asks another server for a profile. *federation.Client
+// satisfies it.
+type RemoteProfiles interface {
+	QueryProfile(ctx context.Context, destination, userID, field string) (map[string]any, error)
+}
+
+// lookupProfile resolves the user ID in the path to an account: a local
+// one from the database, a remote one by asking its server.
 //
 // A user ID on another server is where the federation switch shows up in the
 // Client-Server API: with federation off there is no way to answer and the
 // spec's M_FORBIDDEN is the honest reply, as PLAN.md section 3 requires.
-// With federation on, the lookup itself is Phase 5 work.
 func (c *ClientAPI) lookupProfile(r *http.Request) (storage.Account, error) {
 	userID := r.PathValue("userID")
 
 	localpart, err := c.users.ParseLocalUserID(userID)
 	switch {
 	case errors.Is(err, userapi.ErrForeignServer):
-		if !c.federation.Enabled() {
+		if !c.federation.Enabled() || c.remoteProfile == nil {
 			return storage.Account{}, federation.RejectClientRequest(
 				"looking up the profile of a user on another server")
 		}
-		return storage.Account{}, httputil.NotFound(
-			"Profiles of users on other servers cannot be fetched yet")
+		return c.lookupRemoteProfile(r.Context(), userID)
 	case err != nil:
 		return storage.Account{}, httputil.InvalidParam("%s is not a valid user ID", userID)
 	}
@@ -155,3 +162,34 @@ func profileWriteError(err error) error {
 	}
 	return err
 }
+
+// lookupRemoteProfile asks a user's own server for their profile.
+//
+// Nothing is cached. A profile changes whenever its owner likes, and a
+// cached copy would need invalidating by an EDU this server does not
+// receive; the membership events of the rooms a remote user shares with
+// ours carry their display name already, which is where clients look for
+// it most of the time.
+func (c *ClientAPI) lookupRemoteProfile(ctx context.Context, userID string) (storage.Account, error) {
+	server, err := id.ServerNameOf(userID)
+	if err != nil {
+		return storage.Account{}, httputil.InvalidParam("%s is not a valid user ID", userID)
+	}
+	profile, err := c.remoteProfile.QueryProfile(ctx, server, userID, "")
+	switch {
+	case federation.IsNotFound(err):
+		return storage.Account{}, httputil.NotFound("Profile not found")
+	case err != nil:
+		c.log.Info("remote profile lookup failed", "user_id", userID, "error", err)
+		return storage.Account{}, httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown,
+			"%s could not be asked for the profile", server)
+	}
+	account := storage.Account{}
+	if name, ok := profile["displayname"].(string); ok {
+		account.DisplayName = name
+	}
+	if avatar, ok := profile["avatar_url"].(string); ok {
+		account.AvatarURL = avatar
+	}
+	return account, nil
+}

server/internal/clientapi/profile_test.go

@@ -1,9 +1,13 @@
 package clientapi_test
 
 import (
+	"context"
 	"net/http"
 	"strings"
 	"testing"
+
+	"github.com/dariuszjaniak/warp/server/internal/clientapi"
+	"github.com/dariuszjaniak/warp/server/internal/federation"
 )
 
 const profilePath = "/_matrix/client/v3/profile"
@@ -120,7 +124,8 @@ func TestProfileRejectsMalformedUserID(t *testing.T) {
 // no way to answer, and the spec's M_FORBIDDEN with a clear message is the
 // honest reply.
 func TestProfileOfRemoteUserHonoursTheFederationSwitch(t *testing.T) {
-	ts := newTestServer(t, nil)
+	remote := fakeRemoteProfiles{"@bob:other.example": {"displayname": "Bob Remote"}}
+	ts := newTestServer(t, func(o *clientapi.Options) { o.RemoteProfiles = remote })
 
 	rec := ts.get(profilePath+"/@bob:other.example", "")
 	expectError(t, rec, http.StatusForbidden, "M_FORBIDDEN")
@@ -128,15 +133,33 @@ func TestProfileOfRemoteUserHonoursTheFederationSwitch(t *testing.T) {
 		t.Errorf("the message should say why, got %q", msg)
 	}
 
-	// With the switch on it is no longer a refusal, only something that is
-	// not implemented until Phase 5.
+	// With the switch on the user's own server is asked.
 	if err := ts.settings.SetFederationEnabled(t.Context(), true); err != nil {
 		t.Fatalf("SetFederationEnabled: %v", err)
 	}
-	expectError(t, ts.get(profilePath+"/@bob:other.example", ""),
+	rec = ts.get(profilePath+"/@bob:other.example", "")
+	if rec.Code != http.StatusOK {
+		t.Fatalf("remote profile status = %d (body %s)", rec.Code, rec.Body.String())
+	}
+	if name, _ := jsonBody(t, rec)["displayname"].(string); name != "Bob Remote" {
+		t.Errorf("remote display name = %q", name)
+	}
+	expectError(t, ts.get(profilePath+"/@nobody:other.example", ""),
 		http.StatusNotFound, "M_NOT_FOUND")
 }
 
+// fakeRemoteProfiles answers profile queries from a table, as the remote
+// server would.
+type fakeRemoteProfiles map[string]map[string]any
+
+func (f fakeRemoteProfiles) QueryProfile(_ context.Context, _, userID, _ string) (map[string]any, error) {
+	profile, ok := f[userID]
+	if !ok {
+		return nil, &federation.HTTPError{Status: http.StatusNotFound, Code: "M_NOT_FOUND"}
+	}
+	return profile, nil
+}
+
 func TestProfileRejectsOverlongDisplayName(t *testing.T) {
 	ts := newTestServer(t, nil)
 	acc := ts.register("alice", "superuser")

server/internal/clientapi/rooms.go

@@ -46,6 +46,14 @@ func roomError(err error) error {
 
 	case errors.Is(err, roomserver.ErrFederationDisabled):
 		return httputil.Forbidden("%s", err.Error())
+	case errors.Is(err, roomserver.ErrIncompatibleRoomVersion):
+		return httputil.NewError(http.StatusBadRequest, "M_INCOMPATIBLE_ROOM_VERSION", "%s", err.Error())
+	case errors.Is(err, roomserver.ErrNotInRoom):
+		return httputil.Forbidden("%s", err.Error())
+	case errors.Is(err, roomserver.ErrRemote), errors.Is(err, roomserver.ErrBadEvent):
+		// Another server failed us; the client can do nothing about it but
+		// try again, or try another server.
+		return httputil.NewError(http.StatusBadGateway, httputil.ErrCodeUnknown, "%s", err.Error())
 	case errors.Is(err, roomserver.ErrRejected), errors.Is(err, roomserver.ErrForbidden):
 		return httputil.Forbidden("%s", err.Error())
 	case errors.Is(err, roomserver.ErrInvalidContent):

server/internal/clientapi/sync_test.go

@@ -257,7 +257,10 @@ func TestSyncRejectsAnUnreadableToken(t *testing.T) {
 	ts := newTestServer(t, nil)
 	alice := ts.register("alice", "correct-horse-battery-staple")
 
-	for _, since := range []string{"nonsense", "s-1", "sx", "s1_2_3_4_5_6"} {
+	// "s-1" is not here: a negative event position is a /messages token
+	// from inside backfilled history, and a valid ?since=. Only the event
+	// stream goes below zero.
+	for _, since := range []string{"nonsense", "s1_-1", "sx", "s1_2_3_4_5_6"} {
 		rec := ts.get("/_matrix/client/v3/sync?since="+url.QueryEscape(since), alice.AccessToken)
 		expectError(t, rec, http.StatusBadRequest, "M_INVALID_PARAM")
 	}