package storage_test import ( "database/sql" "errors" "os" "testing" "github.com/dariuszjaniak/warp/server/internal/storage" ) // The sync store and the stream reads of the room store, against every // driver that can run without external services. // migratedSyncDB opens a driver, migrates it and empties both the // roomserver and the sync tables. func migratedSyncDB(t *testing.T, driver string) storage.Database { t.Helper() db := migratedRoomDB(t, driver) resetSyncTables(t, driver) return db } // resetSyncTables empties the sync tables and rewinds their counters, for // the same reason resetRoomTables exists: the PostgreSQL tests share one // database. func resetSyncTables(t *testing.T, driver string) { t.Helper() if driver != storage.DriverPostgres { return } db, err := sql.Open("pgx", os.Getenv("WARP_TEST_POSTGRES_DSN")) if err != nil { t.Fatalf("open postgres to reset the sync tables: %v", err) } defer func() { _ = db.Close() }() if _, err := db.ExecContext(t.Context(), `TRUNCATE sync_receipts, sync_to_device, sync_presence, sync_filters`); err != nil { t.Fatalf("reset the sync tables: %v", err) } if _, err := db.ExecContext(t.Context(), `UPDATE sync_sequences SET value = 0`); err != nil { t.Fatalf("rewind the sync counters: %v", err) } } // writeMessages appends n plausible message events to a room and returns // their IDs in order. The room store never parses an event, so the JSON // only has to be JSON. func writeMessages(t *testing.T, rooms storage.RoomStore, roomID string, ids ...string) { t.Helper() for _, id := range ids { if _, err := rooms.Write(t.Context(), storage.RoomWrite{ Event: newTestEvent(roomID, id, "m.room.message", nil), Extremities: []string{id}, }); err != nil { t.Fatalf("Write %s: %v", id, err) } } } func TestTimelineBoundsAndDirection(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() rooms := migratedSyncDB(t, driver).Rooms() if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } writeMessages(t, rooms, testRoomID, "$a", "$b", "$c", "$d", "$e") top, err := rooms.MaxStreamOrdering(ctx) if err != nil { t.Fatalf("MaxStreamOrdering: %v", err) } if top != 5 { t.Fatalf("MaxStreamOrdering = %d, want 5", top) } for _, tc := range []struct { name string req storage.TimelineRequest wantOrder []string }{ { // Low is exclusive, High inclusive, in both // directions: that is what lets a token be handed back // without re-reading the event it names. name: "forwards from the start", req: storage.TimelineRequest{ RoomID: testRoomID, Direction: storage.DirectionForward, Limit: 10, }, wantOrder: []string{"$a", "$b", "$c", "$d", "$e"}, }, { name: "forwards past a position", req: storage.TimelineRequest{ RoomID: testRoomID, Low: 2, Direction: storage.DirectionForward, Limit: 10, }, wantOrder: []string{"$c", "$d", "$e"}, }, { name: "forwards up to a position", req: storage.TimelineRequest{ RoomID: testRoomID, High: 3, Direction: storage.DirectionForward, Limit: 10, }, wantOrder: []string{"$a", "$b", "$c"}, }, { // The limit trims the far end, so a backwards read // keeps the newest: that is what a timeline is. name: "backwards keeps the newest", req: storage.TimelineRequest{ RoomID: testRoomID, Direction: storage.DirectionBackward, Limit: 2, }, wantOrder: []string{"$e", "$d"}, }, { name: "forwards keeps the oldest", req: storage.TimelineRequest{ RoomID: testRoomID, Direction: storage.DirectionForward, Limit: 2, }, wantOrder: []string{"$a", "$b"}, }, { name: "a window with both bounds", req: storage.TimelineRequest{ RoomID: testRoomID, Low: 1, High: 4, Direction: storage.DirectionBackward, Limit: 10, }, wantOrder: []string{"$d", "$c", "$b"}, }, } { t.Run(tc.name, func(t *testing.T) { events, err := rooms.Timeline(ctx, tc.req) if err != nil { t.Fatalf("Timeline: %v", err) } got := make([]string, 0, len(events)) for _, e := range events { got = append(got, e.EventID) } if len(got) != len(tc.wantOrder) { t.Fatalf("Timeline returned %v, want %v", got, tc.wantOrder) } for i := range got { if got[i] != tc.wantOrder[i] { t.Fatalf("Timeline returned %v, want %v", got, tc.wantOrder) } } }) } }) } } func TestTimelineFilterIsAppliedInSQL(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() rooms := migratedSyncDB(t, driver).Rooms() if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } // Two types and two senders. for _, e := range []storage.StoredEvent{ newTestEvent(testRoomID, "$msg1", "m.room.message", nil), newTestEvent(testRoomID, "$topic", "m.room.topic", stateKey("")), func() storage.StoredEvent { e := newTestEvent(testRoomID, "$msg2", "m.room.message", nil) e.Sender = "@bob:warp.example" return e }(), } { if _, err := rooms.Write(ctx, storage.RoomWrite{ Event: e, Extremities: []string{e.EventID}, }); err != nil { t.Fatalf("Write %s: %v", e.EventID, err) } } for _, tc := range []struct { name string filter storage.TimelineFilter want []string }{ {"types", storage.TimelineFilter{Types: []string{"m.room.message"}}, []string{"$msg1", "$msg2"}}, {"not_types", storage.TimelineFilter{NotTypes: []string{"m.room.message"}}, []string{"$topic"}}, {"senders", storage.TimelineFilter{Senders: []string{"@bob:warp.example"}}, []string{"$msg2"}}, {"not_senders", storage.TimelineFilter{NotSenders: []string{"@bob:warp.example"}}, []string{"$msg1", "$topic"}}, { // A trailing '*' is the spec's wildcard. "a wildcard type", storage.TimelineFilter{Types: []string{"m.room.m*"}}, []string{"$msg1", "$msg2"}, }, { // not_types wins over types for an event matching // both, which the spec requires. "exclusions win", storage.TimelineFilter{ Types: []string{"m.room.message"}, NotTypes: []string{"m.room.message"}, }, nil, }, } { t.Run(tc.name, func(t *testing.T) { events, err := rooms.Timeline(ctx, storage.TimelineRequest{ RoomID: testRoomID, Direction: storage.DirectionForward, Limit: 10, Filter: tc.filter, }) if err != nil { t.Fatalf("Timeline: %v", err) } got := make([]string, 0, len(events)) for _, e := range events { got = append(got, e.EventID) } if len(got) != len(tc.want) { t.Fatalf("Timeline returned %v, want %v", got, tc.want) } for i := range got { if got[i] != tc.want[i] { t.Fatalf("Timeline returned %v, want %v", got, tc.want) } } }) } }) } } func TestTimelineWildcardDoesNotTreatPatternCharactersAsWildcards(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() rooms := migratedSyncDB(t, driver).Rooms() if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } writeMessages(t, rooms, testRoomID, "$msg") // A filter pattern is matched with LIKE, so a '%' or '_' in it // has to be escaped or it would match anything. events, err := rooms.Timeline(ctx, storage.TimelineRequest{ RoomID: testRoomID, Direction: storage.DirectionForward, Limit: 10, Filter: storage.TimelineFilter{Types: []string{"m.room.m%*"}}, }) if err != nil { t.Fatalf("Timeline: %v", err) } if len(events) != 0 { t.Errorf("a '%%' in a filter pattern matched %d events; it must be literal", len(events)) } }) } } func TestRoomsWithEventsNarrowsToWhatChanged(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() rooms := migratedSyncDB(t, driver).Rooms() const other = "!other:warp.example" for _, id := range []string{testRoomID, other} { if err := rooms.CreateRoom(ctx, newTestRoom(id)); err != nil { t.Fatalf("CreateRoom %s: %v", id, err) } } writeMessages(t, rooms, testRoomID, "$a") writeMessages(t, rooms, other, "$b") writeMessages(t, rooms, testRoomID, "$c") // Only the first room has an event after position 2. changed, err := rooms.RoomsWithEvents(ctx, []string{testRoomID, other}, 2, 0) if err != nil { t.Fatalf("RoomsWithEvents: %v", err) } if len(changed) != 1 || changed[0] != testRoomID { t.Errorf("RoomsWithEvents = %v, want [%s]", changed, testRoomID) } // Nothing happened after the top of the stream. quiet, err := rooms.RoomsWithEvents(ctx, []string{testRoomID, other}, 3, 0) if err != nil { t.Fatalf("RoomsWithEvents: %v", err) } if len(quiet) != 0 { t.Errorf("RoomsWithEvents past the top = %v, want nothing", quiet) } }) } } func TestStateAtStreamIsTheStateAfterAPosition(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() rooms := migratedSyncDB(t, driver).Rooms() if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } create := []storage.StateEntry{{Type: "m.room.create", StateKey: "", EventID: "$create"}} if _, err := rooms.Write(ctx, storage.RoomWrite{ Event: newTestEvent(testRoomID, "$create", "m.room.create", stateKey("")), StateDelta: create, Extremities: []string{"$create"}, }); err != nil { t.Fatalf("Write the create event: %v", err) } // A topic, authorised against the state the create event left. if _, err := rooms.Write(ctx, storage.RoomWrite{ Event: newTestEvent(testRoomID, "$topic", "m.room.topic", stateKey("")), StateBefore: create, StateDelta: []storage.StateEntry{{Type: "m.room.topic", StateKey: "", EventID: "$topic"}}, Extremities: []string{"$topic"}, }); err != nil { t.Fatalf("Write the topic: %v", err) } // A message after it, whose state-before includes the topic. if _, err := rooms.Write(ctx, storage.RoomWrite{ Event: newTestEvent(testRoomID, "$msg", "m.room.message", nil), StateBefore: append(create, storage.StateEntry{Type: "m.room.topic", StateKey: "", EventID: "$topic"}), Extremities: []string{"$msg"}, }); err != nil { t.Fatalf("Write the message: %v", err) } for _, tc := range []struct { pos int64 types []string }{ // Before anything happened. {0, nil}, // After the create event: the state is the create event, // which is the snapshot *before* it plus itself. {1, []string{"m.room.create"}}, // After the topic. {2, []string{"m.room.create", "m.room.topic"}}, // After a message, which changes no state. {3, []string{"m.room.create", "m.room.topic"}}, } { entries, err := rooms.StateAtStream(ctx, testRoomID, tc.pos) if err != nil { t.Fatalf("StateAtStream(%d): %v", tc.pos, err) } if len(entries) != len(tc.types) { t.Errorf("StateAtStream(%d) returned %d entries, want %d (%v)", tc.pos, len(entries), len(tc.types), tc.types) continue } have := map[string]bool{} for _, e := range entries { have[e.Type] = true } for _, want := range tc.types { if !have[want] { t.Errorf("StateAtStream(%d) is missing %s", tc.pos, want) } } } }) } } func TestReceiptsOnlyEverAdvance(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() db := migratedSyncDB(t, driver) rooms, sync := db.Rooms(), db.Sync() if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } writeMessages(t, rooms, testRoomID, "$first", "$second") receipt := storage.Receipt{ RoomID: testRoomID, UserID: "@alice:warp.example", ReceiptType: "m.read", EventID: "$second", TimestampMS: 1_700_000_000_000, } pos, moved, err := sync.SetReceipt(ctx, receipt) if err != nil { t.Fatalf("SetReceipt: %v", err) } if !moved || pos == 0 { t.Fatalf("SetReceipt = (%d, %v), want a position and moved", pos, moved) } // Backwards: two of a user's clients read a room at different // speeds, and letting the slower one win would make the unread // marker jump backwards on the other. backwards := receipt backwards.EventID = "$first" back, moved, err := sync.SetReceipt(ctx, backwards) if err != nil { t.Fatalf("SetReceipt backwards: %v", err) } if moved { t.Error("a receipt that would move backwards was written") } if back != pos { t.Errorf("the position moved from %d to %d on a rejected receipt", pos, back) } current, err := sync.Receipts(ctx, []string{testRoomID}, 0, 0) if err != nil { t.Fatalf("Receipts: %v", err) } if len(current) != 1 || current[0].EventID != "$second" { t.Errorf("Receipts = %+v, want the one receipt at $second", current) } }) } } func TestReceiptForAnUnknownEventIsRefused(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() db := migratedSyncDB(t, driver) if err := db.Rooms().CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } _, _, err := db.Sync().SetReceipt(ctx, storage.Receipt{ RoomID: testRoomID, UserID: "@alice:warp.example", ReceiptType: "m.read", EventID: "$nope", }) if !errors.Is(err, storage.ErrNoSuchEvent) { t.Errorf("SetReceipt for an unknown event = %v, want ErrNoSuchEvent", err) } }) } } func TestToDeviceQueueIsPerDeviceAndDrainable(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() sync := migratedSyncDB(t, driver).Sync() top, err := sync.SendToDevice(ctx, []storage.NewToDeviceMessage{ {Localpart: "alice", DeviceID: "ONE", Sender: "@bob:warp.example", Type: "m.room_key", Content: []byte(`{"n":1}`)}, {Localpart: "alice", DeviceID: "TWO", Sender: "@bob:warp.example", Type: "m.room_key", Content: []byte(`{"n":2}`)}, {Localpart: "alice", DeviceID: "ONE", Sender: "@bob:warp.example", Type: "m.room_key", Content: []byte(`{"n":3}`)}, }) if err != nil { t.Fatalf("SendToDevice: %v", err) } if top != 3 { t.Errorf("SendToDevice returned %d, want 3: every message takes a position", top) } one, err := sync.ToDeviceMessages(ctx, "alice", "ONE", 0, 0, 0) if err != nil { t.Fatalf("ToDeviceMessages: %v", err) } if len(one) != 2 { t.Fatalf("device ONE has %d messages, want 2", len(one)) } // Draining one device's queue leaves the other's alone: a // message to two devices is two independent deliveries. if err := sync.DeleteToDeviceMessages(ctx, "alice", "ONE", one[len(one)-1].StreamPos); err != nil { t.Fatalf("DeleteToDeviceMessages: %v", err) } if remaining, err := sync.ToDeviceMessages(ctx, "alice", "ONE", 0, 0, 0); err != nil { t.Fatalf("ToDeviceMessages: %v", err) } else if len(remaining) != 0 { t.Errorf("device ONE still has %d messages after draining", len(remaining)) } if two, err := sync.ToDeviceMessages(ctx, "alice", "TWO", 0, 0, 0); err != nil { t.Fatalf("ToDeviceMessages: %v", err) } else if len(two) != 1 { t.Errorf("device TWO has %d messages, want its own 1", len(two)) } }) } } func TestPresenceStreamMovesOnlyOnARealChange(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() sync := migratedSyncDB(t, driver).Sync() state := storage.PresenceState{ UserID: "@alice:warp.example", Presence: "online", LastActiveMS: 1_000, } first, moved, err := sync.SetPresence(ctx, state) if err != nil { t.Fatalf("SetPresence: %v", err) } if !moved { t.Error("the first presence write must move the stream") } // Every /sync is a write that only refreshes the timestamp. A // stream that moved each time would wake every peer's long // poll, which would sync, which would move it again. state.LastActiveMS = 2_000 again, moved, err := sync.SetPresence(ctx, state) if err != nil { t.Fatalf("SetPresence: %v", err) } if moved { t.Error("a write that refreshes only the timestamp must not move the stream") } if again != first { t.Errorf("the position moved from %d to %d without a change", first, again) } // The timestamp did update, even though the stream did not. read, err := sync.Presence(ctx, state.UserID) if err != nil { t.Fatalf("Presence: %v", err) } if read.LastActiveMS != 2_000 { t.Errorf("last_active_ms = %d, want it refreshed to 2000", read.LastActiveMS) } state.Presence = "unavailable" third, moved, err := sync.SetPresence(ctx, state) if err != nil { t.Fatalf("SetPresence: %v", err) } if !moved || third <= first { t.Errorf("a real change gave (%d, %v), want a higher position and moved", third, moved) } }) } } func TestPresenceOfAnUnknownUser(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { sync := migratedSyncDB(t, driver).Sync() _, err := sync.Presence(t.Context(), "@nobody:warp.example") if !errors.Is(err, storage.ErrNoSuchUser) { t.Errorf("Presence of an unknown user = %v, want ErrNoSuchUser", err) } }) } } func TestFiltersAreStoredVerbatimAndScopedToTheirOwner(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() sync := migratedSyncDB(t, driver).Sync() // Spacing and key order are preserved: a client compares what // it gets back with what it sent. raw := []byte(`{"room":{"timeline":{"limit":5}},"event_fields":["type"]}`) filterID, err := sync.SaveFilter(ctx, "alice", raw, 1_700_000_000_000) if err != nil { t.Fatalf("SaveFilter: %v", err) } if filterID == "" { t.Fatal("SaveFilter returned no ID") } read, err := sync.Filter(ctx, "alice", filterID) if err != nil { t.Fatalf("Filter: %v", err) } if string(read) != string(raw) { t.Errorf("Filter = %s, want it byte for byte: %s", read, raw) } // A filter belongs to the user who uploaded it. if _, err := sync.Filter(ctx, "bob", filterID); !errors.Is(err, storage.ErrNoSuchFilter) { t.Errorf("another user's filter = %v, want ErrNoSuchFilter", err) } if _, err := sync.Filter(ctx, "alice", "nope"); !errors.Is(err, storage.ErrNoSuchFilter) { t.Errorf("an unknown filter = %v, want ErrNoSuchFilter", err) } }) } } func TestStreamPositionsReadsEveryStream(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() db := migratedSyncDB(t, driver) rooms, sync := db.Rooms(), db.Sync() positions, err := sync.StreamPositions(ctx) if err != nil { t.Fatalf("StreamPositions: %v", err) } if positions != (storage.StreamPositions{}) { t.Errorf("a fresh database is at %+v, want every stream at zero", positions) } if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } writeMessages(t, rooms, testRoomID, "$a") if _, _, err := sync.SetReceipt(ctx, storage.Receipt{ RoomID: testRoomID, UserID: "@alice:warp.example", ReceiptType: "m.read", EventID: "$a", }); err != nil { t.Fatalf("SetReceipt: %v", err) } if _, err := sync.SendToDevice(ctx, []storage.NewToDeviceMessage{ {Localpart: "alice", DeviceID: "ONE", Type: "m.room_key", Content: []byte(`{}`)}, }); err != nil { t.Fatalf("SendToDevice: %v", err) } if _, _, err := sync.SetPresence(ctx, storage.PresenceState{ UserID: "@alice:warp.example", Presence: "online", }); err != nil { t.Fatalf("SetPresence: %v", err) } positions, err = sync.StreamPositions(ctx) if err != nil { t.Fatalf("StreamPositions: %v", err) } if positions.PDU != 1 || positions.Receipt != 1 || positions.ToDevice != 1 || positions.Presence != 1 { t.Errorf("StreamPositions = %+v, want every written stream at 1", positions) } }) } } func TestUserMembershipEventsAreInStreamOrder(t *testing.T) { for _, driver := range testDrivers { t.Run(driver, func(t *testing.T) { ctx := t.Context() rooms := migratedSyncDB(t, driver).Rooms() if err := rooms.CreateRoom(ctx, newTestRoom(testRoomID)); err != nil { t.Fatalf("CreateRoom: %v", err) } const alice = "@alice:warp.example" for _, id := range []string{"$join", "$leave", "$rejoin"} { e := newTestEvent(testRoomID, id, "m.room.member", stateKey(alice)) if _, err := rooms.Write(ctx, storage.RoomWrite{ Event: e, Extremities: []string{id}, }); err != nil { t.Fatalf("Write %s: %v", id, err) } } // Somebody else's membership must not be in the answer. other := newTestEvent(testRoomID, "$bob", "m.room.member", stateKey("@bob:warp.example")) if _, err := rooms.Write(ctx, storage.RoomWrite{ Event: other, Extremities: []string{"$bob"}, }); err != nil { t.Fatalf("Write $bob: %v", err) } events, err := rooms.UserMembershipEvents(ctx, testRoomID, alice) if err != nil { t.Fatalf("UserMembershipEvents: %v", err) } want := []string{"$join", "$leave", "$rejoin"} if len(events) != len(want) { t.Fatalf("UserMembershipEvents returned %d events, want %d", len(events), len(want)) } for i, e := range events { if e.EventID != want[i] { t.Errorf("event %d = %s, want %s", i, e.EventID, want[i]) } } }) } }