Warp/server/internal/roomserver/backfill_test.go

package roomserver_test

import (
	"context"
	"fmt"
	"slices"
	"testing"

	"github.com/dariuszjaniak/warp/server/internal/federation"
	"github.com/dariuszjaniak/warp/server/internal/matrix/event"
	"github.com/dariuszjaniak/warp/server/internal/roomserver"
	"github.com/dariuszjaniak/warp/server/internal/storage"
)

// Outbound backfill: a server that joined a room part-way through filling
// in the history before its join when a client scrolls back to it.

func (t transport) Backfill(ctx context.Context, dest, roomID string, from []string, limit int) (federation.Transaction, error) {
	api, err := t.to(dest)
	if err != nil {
		return federation.Transaction{}, err
	}
	pdus, err := api.BackfillForServer(ctx, t.from.name, roomID, from, limit)
	if err != nil {
		return federation.Transaction{}, httpError(err)
	}
	return federation.Transaction{Origin: dest, PDUs: pdus}, nil
}

// scrollBack pages a room's history backwards the way a client scrolling
// up does — /messages from the end, then from each page's `end` token —
// and returns every event ID it was shown, oldest first.
func (p *peer) scrollBack(t *testing.T, roomID, user string, pageSize int) []string {
	t.Helper()
	var ids []string
	var high int64
	for range 100 {
		page, err := p.api.Timeline(t.Context(), roomserver.TimelineRequest{
			RoomID: roomID, User: user, High: high,
			Direction: storage.DirectionBackward, Limit: pageSize, Backfill: true,
		})
		if err != nil {
			t.Fatalf("%s: Timeline: %v", p.name, err)
		}
		page.Events = slices.Clone(page.Events)
		slices.Reverse(page.Events)
		for _, e := range page.Events {
			ids = append(ids, e.EventID)
		}
		if !page.Limited {
			slices.Reverse(ids)
			return ids
		}
		high = page.Start
	}
	t.Fatal("scrolling back never reached the end of the room")
	return nil
}

func TestBackfillFillsInTheHistoryBeforeAJoin(t *testing.T) {
	for _, version := range []string{"10", "11", "12"} {
		t.Run("v"+version, func(t *testing.T) {
			m := newMesh(t, serverA, serverB)
			a, b := m.servers[serverA], m.servers[serverB]

			resp, err := a.api.CreateRoom(t.Context(), roomserver.CreateRoomRequest{
				Creator: userA, Preset: roomserver.PresetPublicChat, Name: "Federated", Version: version,
			})
			if err != nil {
				t.Fatalf("CreateRoom: %v", err)
			}
			roomID := resp.RoomID

			// Enough history that it takes several /backfill requests,
			// and a state change in the middle of it.
			for i := range 25 {
				a.send(t, roomID, userA, fmt.Sprint("before ", i))
			}
			topicID, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{
				RoomID: roomID, Sender: userA, Type: roomserver.TypeTopic, StateKey: ptrTo(""),
				Content: []byte(`{"topic":"history"}`),
			})
			if err != nil {
				t.Fatalf("SendEvent: %v", err)
			}
			for i := range 10 {
				a.send(t, roomID, userA, fmt.Sprint("after the topic ", i))
			}
			a.takeOutbox() // nobody else is in the room yet

			if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{
				RoomID: roomID, User: userB, ServerNames: []string{serverA},
			}); err != nil {
				t.Fatalf("remote join: %v", err)
			}
			a.send(t, roomID, userA, "after the join")
			if failures := m.flush(); len(failures) > 0 {
				t.Fatalf("delivery failures: %v", failures)
			}

			// Before anybody scrolls, B's timeline starts at its join.
			if held := b.timelineIDs(t, roomID); len(held) != 2 {
				t.Fatalf("B's timeline before backfill = %d events, want the join and one message", len(held))
			}
			top, err := b.api.StreamTop(t.Context())
			if err != nil {
				t.Fatalf("StreamTop: %v", err)
			}

			// Scrolling back on B shows the whole room, in A's order.
			want := a.timelineIDs(t, roomID)
			got := b.scrollBack(t, roomID, userB, 10)
			if !slices.Equal(got, want) {
				t.Fatalf("B's scrollback has %d events, A's room %d:\nB: %v\nA: %v", len(got), len(want), got, want)
			}

			// The history is in B's timeline proper, below everything B
			// held, and is not news: the top of the stream — what a live
			// /sync reads from — has not moved.
			create, err := b.store.Event(t.Context(), roomID, want[0])
			if err != nil {
				t.Fatalf("Event(create): %v", err)
			}
			if create.Outlier || create.StreamOrdering >= 0 || create.Type != event.TypeCreate {
				t.Errorf("B's create event: type %s, position %d, outlier %v; want a timeline event below zero",
					create.Type, create.StreamOrdering, create.Outlier)
			}
			if after, err := b.api.StreamTop(t.Context()); err != nil {
				t.Fatalf("StreamTop: %v", err)
			} else if after != top {
				t.Errorf("backfill moved the stream top from %d to %d", top, after)
			}

			// The room's state at a point inside the history is the state
			// then: no topic before the topic was set, the topic after.
			topic, err := b.store.Event(t.Context(), roomID, topicID)
			if err != nil {
				t.Fatalf("Event(topic): %v", err)
			}
			for pos, wantTopic := range map[int64]bool{
				storage.PositionBefore(topic.StreamOrdering): false,
				topic.StreamOrdering:                         true,
			} {
				state, err := b.api.StateAt(t.Context(), roomID, userB, pos)
				if err != nil {
					t.Fatalf("StateAt(%d): %v", pos, err)
				}
				hasTopic, hasName := false, false
				for _, e := range state {
					hasTopic = hasTopic || e.Type == roomserver.TypeTopic
					hasName = hasName || e.Type == roomserver.TypeName
				}
				if hasTopic != wantTopic || !hasName {
					t.Errorf("state at %d: topic %v, name %v; want topic %v and the name", pos, hasTopic, hasName, wantTopic)
				}
			}

			// Scrolling back again asks nobody anything and shows the same.
			if again := b.scrollBack(t, roomID, userB, 7); !slices.Equal(again, want) {
				t.Errorf("a second scrollback differs:\n%v\n%v", again, want)
			}
		})
	}
}

func TestBackfillWithFederationOffShowsWhatIsHeld(t *testing.T) {
	m := newMesh(t, serverA, serverB)
	a, b := m.servers[serverA], m.servers[serverB]
	roomID := a.publicRoom(t, userA)
	for i := range 5 {
		a.send(t, roomID, userA, fmt.Sprint(i))
	}
	a.takeOutbox()
	if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{
		RoomID: roomID, User: userB, ServerNames: []string{serverA},
	}); err != nil {
		t.Fatalf("remote join: %v", err)
	}

	// With the switch off the page is what B has, and no error: the client
	// is not told /messages failed because another server could not be
	// asked.
	m.setDisabled(true)
	got := b.scrollBack(t, roomID, userB, 10)
	if held := b.timelineIDs(t, roomID); !slices.Equal(got, held) {
		t.Errorf("scrollback with federation off = %v, want what B holds: %v", got, held)
	}

	// And once it is back on, the history is there.
	m.setDisabled(false)
	if got := b.scrollBack(t, roomID, userB, 10); !slices.Equal(got, a.timelineIDs(t, roomID)) {
		t.Errorf("scrollback with federation back on = %d events, want A's %d", len(got), len(a.timelineIDs(t, roomID)))
	}
}

func TestBackfillIsNotAskedForUnderJoinedVisibility(t *testing.T) {
	m := newMesh(t, serverA, serverB)
	a, b := m.servers[serverA], m.servers[serverB]
	roomID := a.publicRoom(t, userA)
	if _, err := a.api.SendEvent(t.Context(), roomserver.SendEventRequest{
		RoomID: roomID, Sender: userA, Type: event.TypeHistoryVisibility, StateKey: ptrTo(""),
		Content: []byte(`{"history_visibility":"joined"}`),
	}); err != nil {
		t.Fatalf("SendEvent: %v", err)
	}
	a.send(t, roomID, userA, "before")
	a.takeOutbox()
	if _, err := b.api.JoinRoom(t.Context(), roomserver.JoinRequest{
		RoomID: roomID, User: userB, ServerNames: []string{serverA},
	}); err != nil {
		t.Fatalf("remote join: %v", err)
	}

	// Bob may not see before his join, so there is nothing to fetch for
	// him, and nothing is.
	if got := b.scrollBack(t, roomID, userB, 10); len(got) != 1 {
		t.Errorf("scrollback = %v, want just the join", got)
	}
	if held := b.timelineIDs(t, roomID); len(held) != 1 {
		t.Errorf("B's timeline = %v, want nothing backfilled", held)
	}
}