Warp/server/internal/roomserver/backfill.go

package roomserver

import (
	"context"
	"encoding/json"
	"fmt"
	"slices"
	"sync"

	"github.com/dariuszjaniak/warp/server/internal/matrix/auth"
	"github.com/dariuszjaniak/warp/server/internal/matrix/event"
	"github.com/dariuszjaniak/warp/server/internal/matrix/roomversion"
	"github.com/dariuszjaniak/warp/server/internal/matrix/stateres"
	"github.com/dariuszjaniak/warp/server/internal/storage"
)

// Outbound backfill: asking other servers for a room's history.
//
// A room joined over federation starts, on this server, at the join. The
// join is handed the room's state, which is enough to take part, but none
// of the events that led to it — so a client scrolling up reaches the join
// and stops, in a room that may have years of history on every other
// server in it. Backfill is what fills that in: when /messages pages back
// past the oldest event held, the servers in the room are asked for what
// came before it (GET /backfill), and what they send is checked like any
// other event from another server and stored *below* everything already
// here.
//
// "Below" is the whole difficulty. The stream ordering is the order clients
// are shown a room in, and its counter only goes up, because every event it
// has ever numbered arrived after the ones before it. History arrives last
// and belongs first, so it is numbered by a second counter that runs
// downwards from -1 (storage.RoomStore.WriteBackfill). Nothing streams
// through those positions: a live /sync is "everything after my token",
// which never reaches below zero, so backfilled history reaches clients
// only through /messages, /context and the state at a point in it — which
// is right, because it is not news.
//
// A backfilled event goes through the same checks as one that arrived
// live, but one: it is not checked against the room's current state. Soft
// failure is about refusing events built on a state the room has since
// moved past, and history is by definition built on a state the room has
// since moved past.

// Bounds on what one page of /messages can make this server fetch.
const (
	// backfillMinBatch is the fewest events one /backfill request asks
	// for, however small the page. A request costs a round trip and a
	// state fetch at its far end whatever its size, so asking for a
	// handful at a time would make scrolling a room's history slow for no
	// saving.
	backfillMinBatch = 20

	// backfillRounds is how many /backfill requests one page may make
	// before it is returned short. Events a server sends that fail the
	// checks, or that the page's filter leaves out, are why one round may
	// not be enough.
	backfillRounds = 3

	// backfillFrontierScan is how many of the room's oldest events are
	// read to find where its history stops.
	backfillFrontierScan = 50
)

// batchEvent is an event of a backfill batch that has passed the checks,
// with the state after it: what the events after it in the batch are
// checked against.
type batchEvent struct {
	event      *event.Event
	stateAfter []*event.Event
}

// wantsBackfill reports whether a page of a room's history came back short
// at the bottom of what this server holds, where other servers may hold
// more.
func (r *API) wantsBackfill(req TimelineRequest, visible VisibleRange, page Timeline) bool {
	if !req.Backfill || req.Direction != storage.DirectionBackward || page.Limited || !r.federating() {
		return false
	}
	// Only a window that is open at the bottom — no `to` from the client,
	// and a history visibility that lets the user see before their join —
	// can reach history this server does not have.
	if low, _, ok := visible.Clamp(req.Low, req.High); !ok || low != 0 {
		return false
	}
	// A page that reaches the create event has reached the beginning of
	// the room.
	return len(page.Events) == 0 || page.Events[0].Type != event.TypeCreate
}

// backfillLock serialises the backfills of one room, so that two clients
// scrolling the same room at once do not fetch the same history twice.
// It is not the room's write lock: a backfill spends most of its time
// waiting for other servers, and live events must not wait with it.
func (r *API) backfillLock(roomID string) *sync.Mutex {
	actual, _ := r.backfills.LoadOrStore(roomID, &sync.Mutex{})
	mu, ok := actual.(*sync.Mutex)
	if !ok {
		// Unreachable: nothing else ever stores into this map.
		panic("roomserver: backfill lock map holds something that is not a mutex")
	}
	return mu
}

// backfill asks the servers in a room for the history before the oldest
// event this server holds, and stores what passes below it. It returns how
// many events were added; zero with no error means there was nothing to
// ask for, or nobody had anything.
func (r *API) backfill(ctx context.Context, room storage.Room, rules roomversion.Rules, want int) (int, error) {
	mu := r.backfillLock(room.ID)
	mu.Lock()
	defer mu.Unlock()

	// Worked out under the lock: a backfill that finished while this one
	// waited has moved it.
	frontier, err := r.backfillFrontier(ctx, room.ID, rules)
	if err != nil || len(frontier) == 0 {
		return 0, err
	}
	limit := min(max(want, backfillMinBatch), maxBackfillLimit)

	var lastErr error
	for _, server := range r.candidateServers(ctx, room.ID, nil) {
		if err := r.requireFederation("backfilling a room"); err != nil {
			return 0, err
		}
		txn, err := r.remote.Backfill(ctx, server, room.ID, frontier, limit)
		if err != nil {
			lastErr = remoteError(server, err)
			r.log.Info("a server could not backfill a room", "room_id", room.ID, "server", server, "error", err)
			continue
		}
		added, err := r.acceptBackfill(ctx, server, room, rules, txn.PDUs)
		if err != nil {
			return 0, err
		}
		if added > 0 {
			return added, nil
		}
		// A server that joined after this one did may have none of the
		// history; the next one may.
	}
	return 0, lastErr
}

// backfillFrontier finds where a room's history stops on this server: the
// prev_events of its oldest timeline events that are not themselves in the
// timeline. They are what /backfill is asked to start from.
//
// The oldest events, not only the oldest one: a room's history is a graph,
// and a batch that ends partway down two branches leaves an edge on each.
// Reading a bounded number of them keeps this one query however long the
// room is; the edges it misses are the ones a gap left higher up, which
// are not history.
func (r *API) backfillFrontier(ctx context.Context, roomID string, rules roomversion.Rules) ([]string, error) {
	oldest, err := r.store.Timeline(ctx, storage.TimelineRequest{
		RoomID:    roomID,
		Direction: storage.DirectionForward,
		Limit:     backfillFrontierScan,
	})
	if err != nil {
		return nil, err
	}
	inScan := make(map[string]bool, len(oldest))
	for _, s := range oldest {
		inScan[s.EventID] = true
	}
	var prevs []string
	for _, s := range oldest {
		e, err := parseEvent(s, rules)
		if err != nil {
			return nil, err
		}
		for _, id := range e.PrevEventIDs() {
			if !inScan[id] && !slices.Contains(prevs, id) {
				prevs = append(prevs, id)
			}
		}
	}
	if len(prevs) == 0 {
		return nil, nil
	}

	// An outlier is held without its place in the timeline, so it is still
	// history to ask for; an event in the timeline further up is not.
	held, err := r.store.EventsByID(ctx, roomID, prevs)
	if err != nil {
		return nil, err
	}
	for _, s := range held {
		if !s.Outlier {
			prevs = slices.DeleteFunc(prevs, func(id string) bool { return id == s.EventID })
		}
	}
	return prevs, nil
}

// acceptBackfill checks the events a /backfill answered with, oldest
// first, and stores the ones that pass in one write.
//
// Each goes through the intake's steps 1 to 3 (see federated.go): it
// verifies, it passes the rules against its own auth events, and it passes
// them against the state before it. That state comes from the events
// before it in the same batch where it can, and otherwise from the state
// at its prev_events as the sending server reports it — which, for the
// oldest events of a batch, is a /state_ids request: the price of having a
// state snapshot for every backfilled event, which is what lets a client
// be told the room's state at any point of its history.
//
// An event that fails is left out and logged. It is not an error: the rest
// of the batch may still be good, and a server that sends bad history is
// one this server will not take history from, not one that breaks
// /messages.
func (r *API) acceptBackfill(ctx context.Context, origin string, room storage.Room, rules roomversion.Rules, raws []json.RawMessage) (int, error) {
	var events []*event.Event
	seen := map[string]bool{}
	for _, raw := range raws {
		e, err := parseRemote(raw, rules)
		if err != nil {
			r.log.Debug("ignoring an unparseable backfilled event", "room_id", room.ID, "origin", origin, "error", err)
			continue
		}
		if !inRoom(e, room.ID) || seen[e.EventID()] {
			continue
		}
		seen[e.EventID()] = true
		events = append(events, e)
	}

	// What is already in the timeline needs nothing; an outlier is checked
	// again, because it is about to be given a place in the timeline and
	// a state snapshot, which it never had.
	ids := make([]string, 0, len(events))
	for _, e := range events {
		ids = append(ids, e.EventID())
	}
	held, err := r.store.EventsByID(ctx, room.ID, ids)
	if err != nil {
		return 0, err
	}
	for _, s := range held {
		if !s.Outlier {
			events = slices.DeleteFunc(events, func(e *event.Event) bool { return e.EventID() == s.EventID })
		}
	}
	if len(events) == 0 {
		return 0, nil
	}

	eventsByDepth(events)
	verified := r.verifyAll(ctx, events)
	create, err := r.createEvent(ctx, room, rules)
	if err != nil {
		return 0, err
	}

	batch := map[string]batchEvent{}
	writes := make([]storage.BackfillEvent, 0, len(verified))
	rejected := 0
	for _, e := range verified {
		in := incoming{origin: origin, room: room, rules: rules, event: e, batch: batch}
		stateBefore, err := r.checkBackfilled(ctx, in, create)
		if err != nil {
			rejected++
			r.log.Debug("a backfilled event was not accepted",
				"room_id", room.ID, "origin", origin, "event_id", e.EventID(), "type", e.Type(), "error", err)
			continue
		}
		after := stateres.Sorted(stateBefore)
		if e.IsState() {
			after = append(after, e)
		}
		batch[e.EventID()] = batchEvent{event: e, stateAfter: after}
		writes = append(writes, storage.BackfillEvent{
			Event:       storedOf(room.ID, e),
			StateBefore: stateEntries(stateBefore),
		})
	}
	if rejected > 0 || len(verified) < len(events) {
		r.log.Info("refused events in a backfill",
			"room_id", room.ID, "origin", origin,
			"unverified", len(events)-len(verified), "rejected", rejected)
	}

	if err := r.store.WriteBackfill(ctx, room.ID, writes); err != nil {
		return 0, err
	}
	return len(writes), nil
}

// checkBackfilled runs one backfilled event through the intake's checks and
// returns the state before it.
func (r *API) checkBackfilled(ctx context.Context, in incoming, create *event.Event) (auth.State, error) {
	e := in.event

	authEvents, err := r.authEventsOf(ctx, in)
	if err != nil {
		return nil, err
	}
	if err := auth.CheckAuthEvents(e, authEvents); err != nil {
		return nil, err
	}
	if err := auth.Check(ctx, e, withCreate(auth.NewState(authEvents), create), r.verifier()); err != nil {
		return nil, err
	}

	// The create event is the one event with nothing before it.
	if e.Type() == event.TypeCreate && len(e.PrevEventIDs()) == 0 {
		if create == nil || e.EventID() != create.EventID() {
			return nil, fmt.Errorf("%w: a second create event", ErrBadEvent)
		}
		return auth.State{}, nil
	}

	stateBefore, err := r.stateBeforeIncoming(ctx, in)
	if err != nil {
		return nil, err
	}
	if err := auth.Check(ctx, e, withCreate(stateBefore, create), r.verifier()); err != nil {
		return nil, fmt.Errorf("%w (against the state before it)", err)
	}
	return stateBefore, nil
}