Warp/server/internal/appservice/appservice_test.go

package appservice

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/dariuszjaniak/warp/server/internal/roomserver"
	"github.com/dariuszjaniak/warp/server/internal/storage"
)

const ircRegistration = `
id: irc
url: %URL%
as_token: as-secret
hs_token: hs-secret
sender_localpart: ircbot
rate_limited: false
namespaces:
  users:
    - exclusive: true
      regex: '@_irc_.*'
  aliases:
    - exclusive: false
      regex: '#_irc_.*:hs'
  rooms: []
de.sorunome.msc2409.push_ephemeral: true
`

func parse(t *testing.T, doc, url string) *Registration {
	t.Helper()
	reg, err := ParseRegistration([]byte(strings.ReplaceAll(doc, "%URL%", url)), "hs")
	if err != nil {
		t.Fatalf("ParseRegistration: %v", err)
	}
	return reg
}

func TestParseRegistration(t *testing.T) {
	reg := parse(t, ircRegistration, "http://localhost:9000")
	if reg.ID != "irc" || reg.SenderLocalpart != "ircbot" || reg.RateLimitedUsers() {
		t.Errorf("registration = %+v", reg)
	}

	// The namespace is anchored at the start only, the way Synapse matches.
	ns := reg.Namespaces.Users[0]
	for id, want := range map[string]bool{
		"@_irc_alice:hs":   true,
		"@_irc_:other":     true,
		"@alice_irc_x:hs":  false,
		"@notbot_irc_x:hs": false,
	} {
		if got := ns.Matches(id); got != want {
			t.Errorf("Matches(%q) = %v", id, got)
		}
	}

	// A url of null is a service that receives nothing.
	noURL := parse(t, strings.Replace(ircRegistration, "url: %URL%", "url: null", 1), "")
	if noURL.URL != "" {
		t.Errorf("url: null = %q", noURL.URL)
	}

	for name, doc := range map[string]string{
		"no id":          strings.Replace(ircRegistration, "id: irc", "id: ''", 1),
		"same tokens":    strings.Replace(ircRegistration, "hs_token: hs-secret", "hs_token: as-secret", 1),
		"bad regex":      strings.Replace(ircRegistration, "'@_irc_.*'", "'@_irc_(.*'", 1),
		"bad sender":     strings.Replace(ircRegistration, "sender_localpart: ircbot", "sender_localpart: 'irc bot'", 1),
		"not http":       strings.Replace(ircRegistration, "url: %URL%", "url: ftp://x", 1),
		"no as_token":    strings.Replace(ircRegistration, "as_token: as-secret", "as_token: ''", 1),
		"empty regex":    strings.Replace(ircRegistration, "'@_irc_.*'", "''", 1),
		"not a document": "- just\n- a list\n",
	} {
		if _, err := ParseRegistration([]byte(doc), "hs"); err == nil {
			t.Errorf("%s: accepted", name)
		}
	}
}

func TestRegistry(t *testing.T) {
	irc := parse(t, ircRegistration, "http://localhost:9000")
	other := parse(t, strings.NewReplacer(
		"id: irc", "id: slack",
		"as_token: as-secret", "as_token: as-slack",
		"hs_token: hs-secret", "hs_token: hs-slack",
		"sender_localpart: ircbot", "sender_localpart: slackbot",
		"'@_irc_.*'", "'@_slack_.*'",
		"exclusive: true", "exclusive: false",
	).Replace(ircRegistration), "")

	reg, err := NewRegistry("hs", []*Registration{irc, other})
	if err != nil {
		t.Fatal(err)
	}
	if reg.ByToken("as-secret").ID != "irc" || reg.ByToken("hs-secret") != nil || reg.ByToken("") != nil {
		t.Error("ByToken")
	}
	if reg.ByID("slack").SenderUserID() != "@slackbot:hs" {
		t.Error("ByID / SenderUserID")
	}
	if owner := reg.ExclusiveUserOwner("@_irc_bob:hs"); owner == nil || owner.ID != "irc" {
		t.Error("the exclusive namespace has no owner")
	}
	if reg.ExclusiveUserOwner("@_slack_bob:hs") != nil {
		t.Error("a non-exclusive namespace was claimed")
	}
	if reg.ExclusiveUserOwner("@slackbot:hs") == nil {
		t.Error("a sender is not claimed by its service")
	}
	svc := reg.ByID("irc")
	if !svc.OwnsUser("@ircbot:hs") || !svc.OwnsUser("@_irc_x:hs") || svc.OwnsUser("@_slack_x:hs") {
		t.Error("OwnsUser")
	}
	if !svc.OwnsAlias("#_irc_chan:hs") || svc.ClaimsAliasExclusively("#_irc_chan:hs") {
		t.Error("alias namespaces")
	}
	if owners := reg.UserOwners("@_slack_x:hs"); len(owners) != 0 {
		t.Errorf("a service without a URL is asked about users: %v", owners)
	}

	if _, err := NewRegistry("hs", []*Registration{irc, irc}); err == nil {
		t.Error("a duplicate ID was accepted")
	}
	clash := parse(t, strings.NewReplacer("id: irc", "id: clash", "as_token: as-secret", "as_token: hs-secret",
		"hs_token: hs-secret", "hs_token: other").Replace(ircRegistration), "http://x")
	if _, err := NewRegistry("hs", []*Registration{irc, clash}); err == nil {
		t.Error("an as_token equal to another service's hs_token was accepted")
	}
}

// fakeRooms is a roomserver with a stream held in memory.
type fakeRooms struct {
	mu      sync.Mutex
	events  []roomserver.StreamEvent
	members map[string][]string
	aliases map[string][]string
}

func (f *fakeRooms) add(roomID, eventType, sender string, stateKey *string) {
	f.mu.Lock()
	defer f.mu.Unlock()
	pos := int64(len(f.events) + 1)
	f.events = append(f.events, roomserver.StreamEvent{Position: pos, Event: roomserver.ClientEvent{
		EventID: "$" + string(rune('a'+pos)), RoomID: roomID, Type: eventType, Sender: sender,
		StateKey: stateKey, Content: json.RawMessage(`{}`),
	}})
}

func (f *fakeRooms) StreamTop(context.Context) (int64, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	return int64(len(f.events)), nil
}

func (f *fakeRooms) StreamEvents(_ context.Context, after, upTo int64, limit int) ([]roomserver.StreamEvent, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	var out []roomserver.StreamEvent
	for _, e := range f.events {
		if e.Position > after && (upTo == 0 || e.Position <= upTo) && len(out) < limit {
			out = append(out, e)
		}
	}
	return out, nil
}

func (f *fakeRooms) JoinedUserIDs(_ context.Context, roomID string) ([]string, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.members[roomID], nil
}

func (f *fakeRooms) LocalAliases(_ context.Context, roomID string) ([]string, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.aliases[roomID], nil
}

// memStore is an AppServiceStore in memory.
type memStore struct {
	mu    sync.Mutex
	state map[string]storage.AppServiceState
}

func (m *memStore) AppServiceState(_ context.Context, id string, start int64) (storage.AppServiceState, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	st, ok := m.state[id]
	if !ok {
		st = storage.AppServiceState{ID: id, StreamPos: start}
		m.state[id] = st
	}
	return st, nil
}

func (m *memStore) BeginTransaction(_ context.Context, id string, txnID, to int64) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	st := m.state[id]
	st.TxnID, st.TxnTo = txnID, to
	m.state[id] = st
	return nil
}

func (m *memStore) CompleteTransaction(_ context.Context, id string, to int64) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	st := m.state[id]
	st.StreamPos, st.TxnTo = to, 0
	m.state[id] = st
	return nil
}

// fakeService records the transactions it is sent, and fails the first
// failFirst of them.
type fakeService struct {
	mu        sync.Mutex
	txns      map[string][]string // txn ID -> event IDs
	order     []string
	failFirst int
	auth      []string
}

func (f *fakeService) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.auth = append(f.auth, r.Header.Get("Authorization"))
	switch {
	case strings.HasPrefix(r.URL.Path, "/_matrix/app/v1/transactions/"):
		if f.failFirst > 0 {
			f.failFirst--
			http.Error(w, `{"errcode":"M_UNKNOWN"}`, http.StatusInternalServerError)
			return
		}
		txnID := strings.TrimPrefix(r.URL.Path, "/_matrix/app/v1/transactions/")
		var body struct {
			Events []struct {
				EventID string `json:"event_id"`
			} `json:"events"`
		}
		raw, _ := io.ReadAll(r.Body)
		_ = json.Unmarshal(raw, &body)
		var ids []string
		for _, e := range body.Events {
			ids = append(ids, e.EventID)
		}
		if _, seen := f.txns[txnID]; !seen {
			f.order = append(f.order, txnID)
		}
		f.txns[txnID] = ids
		_, _ = w.Write([]byte(`{}`))
	case r.URL.Path == "/_matrix/app/v1/users/@_irc_known:hs":
		_, _ = w.Write([]byte(`{}`))
	case strings.HasPrefix(r.URL.Path, "/_matrix/app/v1/users/"):
		http.Error(w, `{"errcode":"M_NOT_FOUND"}`, http.StatusNotFound)
	case r.URL.Path == "/_matrix/app/v1/ping":
		_, _ = w.Write([]byte(`{}`))
	default:
		http.NotFound(w, r)
	}
}

func (f *fakeService) delivered() (map[string][]string, []string) {
	f.mu.Lock()
	defer f.mu.Unlock()
	out := map[string][]string{}
	for k, v := range f.txns {
		out[k] = v
	}
	return out, append([]string(nil), f.order...)
}

func strPtr(s string) *string { return &s }

func TestSenderDeliversWhatConcernsTheService(t *testing.T) {
	service := &fakeService{txns: map[string][]string{}, failFirst: 1}
	srv := httptest.NewServer(service)
	defer srv.Close()

	reg, err := NewRegistry("hs", []*Registration{parse(t, ircRegistration, srv.URL)})
	if err != nil {
		t.Fatal(err)
	}
	rooms := &fakeRooms{
		members: map[string][]string{"!bridged:hs": {"@alice:hs", "@_irc_bob:hs"}},
		aliases: map[string][]string{"!aliased:hs": {"#_irc_chan:hs"}},
	}
	// Before the service is first seen: it starts at the top, and is not
	// told about this.
	rooms.add("!other:hs", "m.room.message", "@alice:hs", nil)

	store := &memStore{state: map[string]storage.AppServiceState{}}
	api, err := New(Options{Registry: reg, Store: store, Rooms: rooms})
	if err != nil {
		t.Fatal(err)
	}
	// The first failure is retried after minBackoff; a ping cuts that short.
	api.Start(t.Context())
	defer api.Close()
	waitFor(t, func() bool { return api.Statuses()[0].StreamPos == 1 })

	rooms.add("!other:hs", "m.room.message", "@alice:hs", nil)                    // 2: no
	rooms.add("!other:hs", "m.room.message", "@_irc_carol:hs", nil)               // 3: sender
	rooms.add("!other:hs", "m.room.member", "@alice:hs", strPtr("@_irc_dave:hs")) // 4: target
	rooms.add("!bridged:hs", "m.room.message", "@alice:hs", nil)                  // 5: member
	rooms.add("!aliased:hs", "m.room.message", "@alice:hs", nil)                  // 6: alias
	rooms.add("!other:hs", "m.room.message", "@ircbot:hs", nil)                   // 7: the sender itself
	api.EventPersisted(t.Context(), roomserver.Notification{})

	waitFor(t, func() bool { return api.Statuses()[0].Failures > 0 })
	if _, err := api.Ping(t.Context(), reg.ByID("irc"), "ping-1"); err != nil {
		t.Fatalf("Ping: %v", err)
	}
	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
	defer cancel()
	if err := api.WaitIdle(ctx); err != nil {
		t.Fatal(err)
	}

	txns, order := service.delivered()
	if len(order) != 1 {
		t.Fatalf("transactions = %v", txns)
	}
	got := strings.Join(txns[order[0]], ",")
	if want := "$d,$e,$f,$g,$h"; got != want {
		t.Errorf("delivered %s, want %s", got, want)
	}
	if order[0] != "1" {
		t.Errorf("first transaction ID = %s", order[0])
	}
	status := api.Statuses()[0]
	if status.Failures != 0 || status.EventsDelivered != 5 || status.TxnID != 1 {
		t.Errorf("status = %+v", status)
	}
	for _, a := range service.auth {
		if a != "Bearer hs-secret" {
			t.Errorf("Authorization = %q", a)
		}
	}

	// Queries: yes for a user the service knows, no otherwise.
	if !api.QueryUser(t.Context(), "@_irc_known:hs") || api.QueryUser(t.Context(), "@_irc_unknown:hs") {
		t.Error("QueryUser")
	}
	if api.QueryUser(t.Context(), "@alice:hs") {
		t.Error("a user outside every namespace was queried")
	}
}

func TestSenderResumesATransactionInFlight(t *testing.T) {
	service := &fakeService{txns: map[string][]string{}}
	srv := httptest.NewServer(service)
	defer srv.Close()
	reg, _ := NewRegistry("hs", []*Registration{parse(t, ircRegistration, srv.URL)})

	rooms := &fakeRooms{}
	rooms.add("!r:hs", "m.room.message", "@_irc_a:hs", nil) // 1
	rooms.add("!r:hs", "m.room.message", "@_irc_b:hs", nil) // 2
	rooms.add("!r:hs", "m.room.message", "@_irc_c:hs", nil) // 3

	// An earlier run began transaction 7 over (0, 2] and stopped.
	store := &memStore{state: map[string]storage.AppServiceState{
		"irc": {ID: "irc", StreamPos: 0, TxnID: 7, TxnTo: 2},
	}}
	api, _ := New(Options{Registry: reg, Store: store, Rooms: rooms})
	api.Start(t.Context())
	defer api.Close()

	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
	defer cancel()
	if err := api.WaitIdle(ctx); err != nil {
		t.Fatal(err)
	}
	txns, order := service.delivered()
	if len(order) != 2 || order[0] != "7" || order[1] != "8" {
		t.Fatalf("transactions = %v (%v)", txns, order)
	}
	if strings.Join(txns["7"], ",") != "$b,$c" || strings.Join(txns["8"], ",") != "$d" {
		t.Errorf("transactions = %v", txns)
	}
}

func TestPingErrors(t *testing.T) {
	client := NewClient(200 * time.Millisecond)

	noURL := &Service{Registration: &Registration{ID: "x", HSToken: "t"}}
	if _, err := client.Ping(t.Context(), noURL, ""); !errors.Is(err, ErrNoURL) {
		t.Errorf("no URL: %v", err)
	}

	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		http.Error(w, "nope", http.StatusForbidden)
	}))
	defer bad.Close()
	var status *BadStatusError
	if _, err := client.Ping(t.Context(), &Service{Registration: &Registration{URL: bad.URL, HSToken: "t"}}, "x"); !errors.As(err, &status) || status.Status != http.StatusForbidden {
		t.Errorf("bad status: %v", err)
	}

	slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		select {
		case <-r.Context().Done():
		case <-time.After(2 * time.Second):
		}
	}))
	defer slow.Close()
	if _, err := client.Ping(t.Context(), &Service{Registration: &Registration{URL: slow.URL, HSToken: "t"}}, ""); !errors.Is(err, ErrConnectionTimeout) {
		t.Errorf("slow: %v", err)
	}

	closed := httptest.NewServer(http.NotFoundHandler())
	closedURL := closed.URL
	closed.Close()
	if _, err := client.Ping(t.Context(), &Service{Registration: &Registration{URL: closedURL, HSToken: "t"}}, ""); !errors.Is(err, ErrConnectionFailed) {
		t.Errorf("closed: %v", err)
	}
}

func waitFor(t *testing.T, cond func() bool) {
	t.Helper()
	deadline := time.Now().Add(5 * time.Second)
	for !cond() {
		if time.Now().After(deadline) {
			t.Fatal("timed out")
		}
		time.Sleep(5 * time.Millisecond)
	}
}