tajniak81andClaude Opus 5.5
b6e82c3633
The chat list and message notifications now say "Video call" or "Voice call" for a call's ring (org.matrix.msc4075.rtc.notification) and "Call declined" for its decline, as the web client does. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1045 lines
33 KiB
Dart
1045 lines
33 KiB
Dart
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
import 'dart:math';
|
|
|
|
import 'package:flutter/foundation.dart';
|
|
import 'package:http/http.dart' as http;
|
|
import 'package:livekit_client/livekit_client.dart' as lk;
|
|
import 'package:matrix/matrix.dart';
|
|
|
|
import 'call_service.dart';
|
|
import 'notifications.dart';
|
|
import 'package:webrtc_interface/webrtc_interface.dart' as rtc;
|
|
import 'package:flutter_webrtc/flutter_webrtc.dart'
|
|
show Helper, frameCryptorFactory;
|
|
|
|
/// Voice and video calls: MatrixRTC (MSC4143) with a LiveKit SFU, the way
|
|
/// the web client and Element Call do them, so the three can share a call.
|
|
///
|
|
/// Who is in a room's call is room state: one
|
|
/// `org.matrix.msc3401.call.member` event per device, state key
|
|
/// `_@user:server_DEVICE_m.call`. Joining sends ours, after scheduling an
|
|
/// empty one with the homeserver (a delayed event, MSC4140) that is pushed
|
|
/// back every few seconds while the call lasts: if the app dies or loses
|
|
/// its connection, the server sends it and the device drops out.
|
|
///
|
|
/// The media goes through LiveKit, whose token the LiveKit JWT service
|
|
/// hands out for an OpenID token from the homeserver. In an encrypted room
|
|
/// every member makes its own media key and sends it to the others over
|
|
/// encrypted to-device messages (`io.element.call.encryption_keys`); LiveKit
|
|
/// encrypts each frame with it.
|
|
///
|
|
/// Starting a call in a room of two rings the other person
|
|
/// (`org.matrix.msc4075.rtc.notification`); declining answers it
|
|
/// (`org.matrix.msc4310.rtc.decline`).
|
|
///
|
|
/// While this device is in a call, a foreground service on Android
|
|
/// ([CallService]) keeps it going with the app in the background, and lets
|
|
/// it share the screen.
|
|
///
|
|
/// One call at a time.
|
|
class Calls extends ChangeNotifier {
|
|
Calls(this.client) {
|
|
_subs
|
|
..add(client.onSync.stream.listen((_) => _refresh()))
|
|
..add(client.onTimelineEvent.stream.listen(_onTimelineEvent))
|
|
..add(client.onToDeviceEvent.stream.listen(_onToDevice));
|
|
CallService.listen(onHangUp: leave, onOpen: () => _openRequests.add(null));
|
|
_expiryTimer = Timer.periodic(
|
|
const Duration(seconds: 30),
|
|
(_) => _refresh(),
|
|
);
|
|
_refresh();
|
|
}
|
|
|
|
final Client client;
|
|
final _subs = <StreamSubscription>[];
|
|
Timer? _expiryTimer;
|
|
|
|
static const memberType = 'org.matrix.msc3401.call.member';
|
|
static const notificationType = 'org.matrix.msc4075.rtc.notification';
|
|
static const declineType = 'org.matrix.msc4310.rtc.decline';
|
|
static const keysType = 'io.element.call.encryption_keys';
|
|
|
|
/// What the room list and a message notification say for a call's ring
|
|
/// or its decline, which the SDK only knows as "Unknown event"; null for
|
|
/// any other event.
|
|
static String? summary(Event event, {required bool withSender}) {
|
|
final String text;
|
|
if (event.type == notificationType) {
|
|
text = event.content['m.call.intent'] == 'audio'
|
|
? 'Voice call'
|
|
: 'Video call';
|
|
} else if (event.type == declineType) {
|
|
text = 'Call declined';
|
|
} else {
|
|
return null;
|
|
}
|
|
if (!withSender) return text;
|
|
return '${event.senderFromMemoryOrFallback.calcDisplayname()}: $text';
|
|
}
|
|
|
|
/// How long a call rings, unless the notification says less.
|
|
static const ringLifetime = Duration(seconds: 60);
|
|
|
|
/// The most a received notification may ring for, whatever it says.
|
|
static const maxRing = Duration(seconds: 120);
|
|
|
|
/// A membership lasts this long unless renewed; calls are shorter.
|
|
static const membershipExpiry = Duration(hours: 4);
|
|
|
|
static const _delayedLeave = Duration(seconds: 8);
|
|
static const _delayedLeaveRestart = Duration(seconds: 5);
|
|
|
|
// --- What the screens show --------------------------------------------
|
|
|
|
/// The room whose call this device is in.
|
|
String? roomId;
|
|
CallPhase? phase;
|
|
bool video = false;
|
|
bool encrypted = false;
|
|
lk.Room? livekit;
|
|
|
|
/// We rang someone and they haven't come yet.
|
|
bool ringingOut = false;
|
|
|
|
/// A call ringing for us.
|
|
IncomingCall? incoming;
|
|
|
|
/// When we joined, for the call's length.
|
|
DateTime? joinedAt;
|
|
|
|
/// How many people (not devices) are in each room's call.
|
|
final active = <String, int>{};
|
|
|
|
/// Errors to show once, then cleared.
|
|
final errors = StreamController<String>.broadcast();
|
|
|
|
final _openRequests = StreamController<void>.broadcast();
|
|
|
|
/// The ongoing call notification was tapped: show the call.
|
|
Stream<void> get openRequests => _openRequests.stream;
|
|
|
|
bool get inCall => roomId != null;
|
|
|
|
int inCallCount(String roomId) => active[roomId] ?? 0;
|
|
|
|
// --- Memberships ------------------------------------------------------
|
|
|
|
/// The live memberships of [room]'s call, one per device.
|
|
List<CallMember> memberships(Room room) {
|
|
final events = room.states[memberType]?.values ?? const [];
|
|
final now = DateTime.now();
|
|
final out = <CallMember>[];
|
|
for (final e in events) {
|
|
final m = CallMember.fromState(e);
|
|
if (m != null && m.expiresAt.isAfter(now)) out.add(m);
|
|
}
|
|
out.sort((a, b) => a.createdAt.compareTo(b.createdAt));
|
|
return out;
|
|
}
|
|
|
|
void _refresh() {
|
|
var changed = false;
|
|
final seen = <String>{};
|
|
for (final room in client.rooms) {
|
|
if (room.membership != Membership.join) continue;
|
|
final users = memberships(room).map((m) => m.userId).toSet();
|
|
final n = users.length;
|
|
if (n > 0) {
|
|
seen.add(room.id);
|
|
if (active[room.id] != n) {
|
|
active[room.id] = n;
|
|
changed = true;
|
|
}
|
|
}
|
|
}
|
|
for (final id in active.keys.toList()) {
|
|
if (!seen.contains(id)) {
|
|
active.remove(id);
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
// The caller gave up, or someone else answered for us.
|
|
final ring = incoming;
|
|
if (ring != null && inCallCount(ring.roomId) == 0) {
|
|
_clearIncoming();
|
|
changed = true;
|
|
}
|
|
final rid = roomId;
|
|
if (rid != null) {
|
|
final room = client.getRoomById(rid);
|
|
if (room != null) {
|
|
final members = memberships(room);
|
|
final people = members.map((m) => m.userId).toSet().length;
|
|
// Someone came: stop ringing them.
|
|
if (ringingOut && people > 1) {
|
|
ringingOut = false;
|
|
changed = true;
|
|
}
|
|
if (encrypted) _shareKeyWithNewDevices(room, members);
|
|
// Everyone else left: there's no one to talk to, so hang up too.
|
|
if (people > 1) {
|
|
_hadCompany = true;
|
|
} else if (_hadCompany) {
|
|
errors.add('The call ended.');
|
|
leave();
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
if (changed) notifyListeners();
|
|
}
|
|
|
|
// --- Ringing ------------------------------------------------------------
|
|
|
|
final _seenEvents = <String>{};
|
|
|
|
void _onTimelineEvent(Event event) {
|
|
if (event.type != notificationType && event.type != declineType) return;
|
|
if (!_seenEvents.add(event.eventId)) return;
|
|
final me = client.userID;
|
|
final room = event.room;
|
|
|
|
if (event.type == notificationType) {
|
|
if (event.senderId == me || roomId == room.id || inCall) return;
|
|
final mentions = event.content.tryGetMap<String, Object?>('m.mentions');
|
|
final userIds = mentions?.tryGetList<String>('user_ids') ?? const [];
|
|
if (mentions?['room'] != true && !userIds.contains(me)) return;
|
|
// A ring for a call of two; in a bigger room the banner says enough.
|
|
if (event.content['notification_type'] != 'ring' &&
|
|
_otherOfTwo(room) == null) {
|
|
return;
|
|
}
|
|
final senderTs = event.content.tryGet<int>('sender_ts');
|
|
final sentAt = senderTs == null
|
|
? event.originServerTs
|
|
: DateTime.fromMillisecondsSinceEpoch(
|
|
min(senderTs, event.originServerTs.millisecondsSinceEpoch),
|
|
);
|
|
final lifetimeMs = min(
|
|
event.content.tryGet<int>('lifetime') ?? ringLifetime.inMilliseconds,
|
|
maxRing.inMilliseconds,
|
|
);
|
|
final expiresAt = sentAt.add(Duration(milliseconds: lifetimeMs));
|
|
if (!expiresAt.isAfter(DateTime.now())) return;
|
|
final caller = event.senderFromMemoryOrFallback;
|
|
incoming = IncomingCall(
|
|
roomId: room.id,
|
|
eventId: event.eventId,
|
|
callerId: event.senderId,
|
|
callerName: caller.calcDisplayname(),
|
|
callerAvatar: caller.avatarUrl,
|
|
roomName: room.getLocalizedDisplayname(),
|
|
video: event.content['m.call.intent'] != 'audio',
|
|
expiresAt: expiresAt,
|
|
);
|
|
_ringTimer?.cancel();
|
|
_ringTimer = Timer(expiresAt.difference(DateTime.now()), () {
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
});
|
|
Notifications.ring(
|
|
roomId: room.id,
|
|
caller: incoming!.callerName,
|
|
video: incoming!.video,
|
|
);
|
|
notifyListeners();
|
|
return;
|
|
}
|
|
|
|
// A decline.
|
|
final target = event.relationshipEventId;
|
|
if (target == null) return;
|
|
if (event.senderId == me && incoming?.eventId == target) {
|
|
// Declined on another of our devices.
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
} else if (target == _sentNotification && ringingOut && room.id == roomId) {
|
|
errors.add(
|
|
'${event.senderFromMemoryOrFallback.calcDisplayname()} declined the call.',
|
|
);
|
|
leave();
|
|
}
|
|
}
|
|
|
|
Timer? _ringTimer;
|
|
|
|
void _clearIncoming() {
|
|
_ringTimer?.cancel();
|
|
if (incoming != null) Notifications.stopRinging();
|
|
incoming = null;
|
|
}
|
|
|
|
/// Answers the call ringing for us.
|
|
Future<void> answer({bool withVideo = true}) async {
|
|
final call = incoming;
|
|
if (call == null) return;
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
await join(call.roomId, video: withVideo && call.video);
|
|
}
|
|
|
|
/// Declines the call ringing for us.
|
|
Future<void> decline() async {
|
|
final call = incoming;
|
|
if (call == null) return;
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
try {
|
|
await client.getRoomById(call.roomId)?.sendEvent({
|
|
'm.relates_to': {
|
|
'rel_type': RelationshipTypes.reference,
|
|
'event_id': call.eventId,
|
|
},
|
|
}, type: declineType);
|
|
} catch (e) {
|
|
Logs().w('Could not decline the call', e);
|
|
}
|
|
}
|
|
|
|
/// The other person in a room of two.
|
|
String? _otherOfTwo(Room room) {
|
|
if ((room.summary.mJoinedMemberCount ?? room.getParticipants().length) !=
|
|
2) {
|
|
return null;
|
|
}
|
|
return room
|
|
.getParticipants([Membership.join])
|
|
.map((u) => u.id)
|
|
.where((id) => id != client.userID)
|
|
.firstOrNull;
|
|
}
|
|
|
|
// --- Joining ------------------------------------------------------------
|
|
|
|
String? _sentNotification;
|
|
|
|
/// Someone else has been in our call since we joined.
|
|
bool _hadCompany = false;
|
|
String? _delayId;
|
|
Timer? _delayTimer;
|
|
Timer? _ringOutTimer;
|
|
lk.BaseKeyProvider? _keyProvider;
|
|
lk.EventsListener<lk.RoomEvent>? _lkListener;
|
|
|
|
/// Our media key, its index, and who has it: the membership of each
|
|
/// device we sent it to, by when it was made, so a device that leaves and
|
|
/// comes back gets it again.
|
|
Uint8List? _key;
|
|
final _keyIndex = 0;
|
|
final _keySentTo = <String, DateTime>{};
|
|
|
|
/// Remote participants whose media key has arrived.
|
|
final _keyed = <String>{};
|
|
|
|
/// Keys that came before LiveKit was ready, by identity.
|
|
final _pendingKeys = <String, (Uint8List, int)>{};
|
|
|
|
String get _deviceId => client.deviceID!;
|
|
String get _identity => '${client.userID}:$_deviceId';
|
|
|
|
String _stateKey(Room room) {
|
|
final key = '${client.userID}_${_deviceId}_m.call';
|
|
// Room versions with MSC3757/3779 own-state keys take it bare.
|
|
final version = room
|
|
.getState(EventTypes.RoomCreate)
|
|
?.content
|
|
.tryGet<String>('room_version');
|
|
return version != null &&
|
|
RegExp(r'^org\.matrix\.msc(3757|3779)\b').hasMatch(version)
|
|
? key
|
|
: '_$key';
|
|
}
|
|
|
|
/// Joins the call in [roomId], or starts one. Starting a call in a room
|
|
/// of two rings the other person.
|
|
Future<void> join(String roomId, {required bool video}) async {
|
|
if (this.roomId == roomId) return;
|
|
if (inCall) await leave();
|
|
final room = client.getRoomById(roomId);
|
|
if (room == null) return;
|
|
if (incoming?.roomId == roomId) _clearIncoming();
|
|
|
|
this.roomId = roomId;
|
|
this.video = video;
|
|
phase = CallPhase.joining;
|
|
encrypted = room.encrypted;
|
|
_sentNotification = null;
|
|
_hadCompany = false;
|
|
_keySentTo.clear();
|
|
notifyListeners();
|
|
|
|
final others = memberships(room);
|
|
final ringing = others.isEmpty && _otherOfTwo(room) != null;
|
|
try {
|
|
final transport = await _chooseTransport(others);
|
|
final (url, jwt) = await _livekitToken(transport, roomId);
|
|
if (this.roomId != roomId) return;
|
|
|
|
if (encrypted) {
|
|
_keyProvider = await _makeKeyProvider();
|
|
_key = _randomKey();
|
|
await _keyProvider!.setRawKey(
|
|
_key!,
|
|
participantId: _identity,
|
|
keyIndex: _keyIndex,
|
|
);
|
|
for (final e in _pendingKeys.entries) {
|
|
await _keyProvider!.setRawKey(
|
|
e.value.$1,
|
|
participantId: e.key,
|
|
keyIndex: e.value.$2,
|
|
);
|
|
}
|
|
_pendingKeys.clear();
|
|
}
|
|
|
|
final lkRoom = lk.Room(
|
|
roomOptions: lk.RoomOptions(
|
|
adaptiveStream: true,
|
|
dynacast: true,
|
|
e2eeOptions: _keyProvider == null
|
|
? null
|
|
: lk.E2EEOptions(keyProvider: _keyProvider!),
|
|
),
|
|
);
|
|
livekit = lkRoom;
|
|
lkRoom.addListener(_onLivekit);
|
|
_lkListener = lkRoom.createListener()
|
|
..on<lk.RoomReconnectingEvent>((_) {
|
|
phase = CallPhase.reconnecting;
|
|
notifyListeners();
|
|
})
|
|
..on<lk.RoomReconnectedEvent>((_) {
|
|
phase = CallPhase.connected;
|
|
notifyListeners();
|
|
})
|
|
..on<lk.RoomDisconnectedEvent>((event) {
|
|
if (livekit != lkRoom) return;
|
|
if (event.reason != lk.DisconnectReason.clientInitiated) {
|
|
errors.add('You were disconnected from the call.');
|
|
}
|
|
leave();
|
|
});
|
|
|
|
// The Matrix side: a delayed leave first, then ourselves.
|
|
await _scheduleDelayedLeave(room);
|
|
final membershipEventId = await client.setRoomStateWithKey(
|
|
roomId,
|
|
memberType,
|
|
_stateKey(room),
|
|
_myMembership(transport, video),
|
|
);
|
|
if (encrypted) _shareKeyWithNewDevices(room, memberships(room));
|
|
if (ringing || others.isEmpty) {
|
|
_sendNotification(room, membershipEventId, ring: ringing, video: video);
|
|
}
|
|
if (ringing) {
|
|
ringingOut = true;
|
|
_ringOutTimer = Timer(ringLifetime, () {
|
|
if (!ringingOut || this.roomId != roomId) return;
|
|
errors.add('No answer.');
|
|
leave();
|
|
});
|
|
}
|
|
|
|
await lkRoom.connect(url, jwt);
|
|
if (this.roomId != roomId) {
|
|
await lkRoom.disconnect();
|
|
return;
|
|
}
|
|
if (_keyProvider != null) await lkRoom.e2eeManager?.setEnabled(true);
|
|
phase = CallPhase.connected;
|
|
joinedAt = DateTime.now();
|
|
notifyListeners();
|
|
|
|
try {
|
|
await lkRoom.localParticipant?.setMicrophoneEnabled(true);
|
|
} catch (e) {
|
|
errors.add("Can't use the microphone: $e");
|
|
}
|
|
if (video) {
|
|
try {
|
|
await lkRoom.localParticipant?.setCameraEnabled(true);
|
|
} catch (e) {
|
|
errors.add("Can't use the camera: $e");
|
|
}
|
|
}
|
|
// Now that the microphone and camera are asked for, the service can
|
|
// declare them.
|
|
_mediaStarted = true;
|
|
notifyListeners();
|
|
} catch (e, s) {
|
|
Logs().e('Joining the call failed', e, s);
|
|
errors.add(e is CallException ? e.message : 'The call failed: $e');
|
|
await leave();
|
|
}
|
|
}
|
|
|
|
void _onLivekit() => notifyListeners();
|
|
|
|
/// Leaves the call this device is in.
|
|
Future<void> leave() async {
|
|
final rid = roomId;
|
|
final lkRoom = livekit;
|
|
roomId = null;
|
|
livekit = null;
|
|
phase = null;
|
|
joinedAt = null;
|
|
ringingOut = false;
|
|
_hadCompany = false;
|
|
_mediaStarted = false;
|
|
_sharingScreen = false;
|
|
_ringOutTimer?.cancel();
|
|
_delayTimer?.cancel();
|
|
_keyProvider = null;
|
|
_key = null;
|
|
_keySentTo.clear();
|
|
_keyed.clear();
|
|
_pendingKeys.clear();
|
|
final delayId = _delayId;
|
|
_delayId = null;
|
|
notifyListeners();
|
|
|
|
lkRoom?.removeListener(_onLivekit);
|
|
await _lkListener?.dispose();
|
|
_lkListener = null;
|
|
try {
|
|
await lkRoom?.disconnect();
|
|
} catch (_) {}
|
|
if (rid != null) {
|
|
final room = client.getRoomById(rid);
|
|
if (room != null) {
|
|
try {
|
|
await client.setRoomStateWithKey(
|
|
rid,
|
|
memberType,
|
|
_stateKey(room),
|
|
{},
|
|
);
|
|
} catch (e) {
|
|
Logs().w('Could not send the leave; the delayed one will', e);
|
|
}
|
|
}
|
|
}
|
|
if (delayId != null) {
|
|
try {
|
|
await _delayedEventAction(delayId, 'cancel');
|
|
} catch (_) {}
|
|
}
|
|
_refresh();
|
|
}
|
|
|
|
Map<String, Object?> _myMembership(
|
|
Map<String, Object?> transport,
|
|
bool video,
|
|
) => {
|
|
'application': 'm.call',
|
|
'call_id': '',
|
|
'scope': 'm.room',
|
|
'device_id': _deviceId,
|
|
'membershipID': _identity,
|
|
'expires': membershipExpiry.inMilliseconds,
|
|
'm.call.intent': video ? 'video' : 'audio',
|
|
'focus_active': {'type': 'livekit', 'focus_selection': 'multi_sfu'},
|
|
'foci_preferred': [transport],
|
|
};
|
|
|
|
void _sendNotification(
|
|
Room room,
|
|
String membershipEventId, {
|
|
required bool ring,
|
|
required bool video,
|
|
}) {
|
|
room
|
|
.sendEvent({
|
|
'slot_id': 'm.call#ROOM',
|
|
'm.mentions': {'user_ids': <String>[], 'room': true},
|
|
'notification_type': ring ? 'ring' : 'notification',
|
|
'm.relates_to': {
|
|
'event_id': membershipEventId,
|
|
'rel_type': RelationshipTypes.reference,
|
|
},
|
|
'sender_ts': DateTime.now().millisecondsSinceEpoch,
|
|
'lifetime': ringLifetime.inMilliseconds,
|
|
'msc4354_sticky_key': 'm.call#ROOM',
|
|
'm.call.intent': video ? 'video' : 'audio',
|
|
}, type: notificationType)
|
|
.then((id) => _sentNotification = id)
|
|
.catchError((Object e) {
|
|
Logs().w('Could not send the call notification', e);
|
|
return null;
|
|
});
|
|
}
|
|
|
|
// --- Delayed leave (MSC4140) ----------------------------------------------
|
|
|
|
Future<void> _scheduleDelayedLeave(Room room) async {
|
|
try {
|
|
final response = await client.request(
|
|
RequestType.PUT,
|
|
'/client/v3/rooms/${Uri.encodeComponent(room.id)}/state/'
|
|
'${Uri.encodeComponent(memberType)}/${Uri.encodeComponent(_stateKey(room))}',
|
|
query: {'org.matrix.msc4140.delay': '${_delayedLeave.inMilliseconds}'},
|
|
data: <String, Object?>{},
|
|
);
|
|
final id = response['delay_id'];
|
|
if (id is! String) return;
|
|
_delayId = id;
|
|
_delayTimer = Timer.periodic(_delayedLeaveRestart, (_) {
|
|
final current = _delayId;
|
|
if (current == null) return;
|
|
_delayedEventAction(current, 'restart').catchError((Object e) {
|
|
Logs().w('Could not push back the delayed leave', e);
|
|
});
|
|
});
|
|
} catch (e) {
|
|
// Without it, a crash leaves us in the call until our membership
|
|
// expires; the call itself still works.
|
|
Logs().w('The server has no delayed events', e);
|
|
}
|
|
}
|
|
|
|
Future<void> _delayedEventAction(String id, String action) => client.request(
|
|
RequestType.POST,
|
|
'/client/unstable/org.matrix.msc4140/delayed_events/'
|
|
'${Uri.encodeComponent(id)}/$action',
|
|
data: <String, Object?>{},
|
|
);
|
|
|
|
// --- LiveKit --------------------------------------------------------------
|
|
|
|
/// The LiveKit JWT service: the one the call already uses, else this
|
|
/// server's.
|
|
Future<Map<String, Object?>> _chooseTransport(List<CallMember> others) async {
|
|
for (final m in others) {
|
|
final t = m.transport;
|
|
if (t != null) return {...t, 'livekit_alias': roomId};
|
|
}
|
|
List? foci;
|
|
try {
|
|
final response = await client.request(
|
|
RequestType.GET,
|
|
'/client/unstable/org.matrix.msc4143/rtc/transports',
|
|
);
|
|
foci = response['rtc_transports'] as List?;
|
|
} catch (_) {
|
|
try {
|
|
final wellKnown = await http.get(
|
|
client.homeserver!.resolve('/.well-known/matrix/client'),
|
|
);
|
|
final json = jsonDecode(wellKnown.body);
|
|
foci = json is Map
|
|
? json['org.matrix.msc4143.rtc_foci'] as List?
|
|
: null;
|
|
} catch (_) {}
|
|
}
|
|
final focus = foci
|
|
?.whereType<Map>()
|
|
.where(
|
|
(f) => f['type'] == 'livekit' && f['livekit_service_url'] is String,
|
|
)
|
|
.firstOrNull;
|
|
if (focus == null) {
|
|
throw const CallException("This server doesn't offer calls.");
|
|
}
|
|
return {
|
|
'type': 'livekit',
|
|
'livekit_service_url': focus['livekit_service_url'],
|
|
'livekit_alias': roomId,
|
|
};
|
|
}
|
|
|
|
/// A LiveKit URL and token from the JWT service, for an OpenID token.
|
|
Future<(String, String)> _livekitToken(
|
|
Map<String, Object?> transport,
|
|
String roomId,
|
|
) async {
|
|
final openid = await client.requestOpenIdToken(client.userID!, {});
|
|
final service = (transport['livekit_service_url'] as String).replaceFirst(
|
|
RegExp(r'/$'),
|
|
'',
|
|
);
|
|
final response = await http.post(
|
|
Uri.parse('$service/sfu/get'),
|
|
headers: {'Content-Type': 'application/json'},
|
|
body: jsonEncode({
|
|
'room': roomId,
|
|
'openid_token': openid.toJson(),
|
|
'device_id': _deviceId,
|
|
}),
|
|
);
|
|
if (response.statusCode != 200) {
|
|
throw CallException(
|
|
'The call service refused to let you in (${response.statusCode}).',
|
|
);
|
|
}
|
|
final body = jsonDecode(response.body);
|
|
if (body is! Map || body['url'] is! String || body['jwt'] is! String) {
|
|
throw const CallException('The call service gave an unusable answer.');
|
|
}
|
|
return (body['url'] as String, body['jwt'] as String);
|
|
}
|
|
|
|
/// Per-participant keys, no ratcheting, keys derived with HKDF: the
|
|
/// settings of the web client and Element Call, whose livekit-client
|
|
/// imports keys as HKDF material.
|
|
Future<lk.BaseKeyProvider> _makeKeyProvider() async {
|
|
final options = rtc.KeyProviderOptions(
|
|
sharedKey: false,
|
|
ratchetSalt: Uint8List.fromList(utf8.encode('LKFrameEncryptionKey')),
|
|
ratchetWindowSize: 0,
|
|
uncryptedMagicBytes: Uint8List.fromList(utf8.encode('LK-ROCKS')),
|
|
failureTolerance: -1,
|
|
keyRingSize: 256,
|
|
keyDerivationAlgorithm: rtc.KeyDerivationAlgorithm.kHKDF,
|
|
);
|
|
final provider = await frameCryptorFactory.createDefaultKeyProvider(
|
|
options,
|
|
);
|
|
return lk.BaseKeyProvider(provider, options);
|
|
}
|
|
|
|
// --- Media keys -----------------------------------------------------------
|
|
|
|
Uint8List _randomKey() {
|
|
final random = Random.secure();
|
|
return Uint8List.fromList(List.generate(16, (_) => random.nextInt(256)));
|
|
}
|
|
|
|
/// Sends our media key to every device in the call that hasn't had it.
|
|
Future<void> _shareKeyWithNewDevices(
|
|
Room room,
|
|
List<CallMember> members,
|
|
) async {
|
|
final key = _key;
|
|
if (key == null) return;
|
|
final targets = members
|
|
.where(
|
|
(m) =>
|
|
!(m.userId == client.userID && m.deviceId == _deviceId) &&
|
|
_keySentTo['${m.userId}:${m.deviceId}'] != m.createdAt,
|
|
)
|
|
.toList();
|
|
if (targets.isEmpty) return;
|
|
for (final m in targets) {
|
|
_keySentTo['${m.userId}:${m.deviceId}'] = m.createdAt;
|
|
}
|
|
try {
|
|
DeviceKeys? known(CallMember m) =>
|
|
client.userDeviceKeys[m.userId]?.deviceKeys[m.deviceId];
|
|
// A device signed in since we last looked isn't in our list yet:
|
|
// without asking, it would never get the key, and hear and see
|
|
// nothing of us.
|
|
final unknown = targets.where((m) => known(m) == null);
|
|
if (unknown.isNotEmpty) {
|
|
await client.updateUserDeviceKeys(
|
|
additionalUsers: unknown.map((m) => m.userId).toSet(),
|
|
);
|
|
}
|
|
final devices = <DeviceKeys>[];
|
|
for (final m in targets) {
|
|
final d = known(m);
|
|
if (d != null) {
|
|
devices.add(d);
|
|
} else {
|
|
// Try again on the next sync.
|
|
_keySentTo.remove('${m.userId}:${m.deviceId}');
|
|
Logs().w('No keys for ${m.userId}:${m.deviceId}; the call key waits');
|
|
}
|
|
}
|
|
if (devices.isEmpty) return;
|
|
await client.sendToDeviceEncrypted(devices, keysType, {
|
|
'keys': {'index': _keyIndex, 'key': base64Encode(key)},
|
|
'room_id': room.id,
|
|
'member': {'claimed_device_id': _deviceId, 'id': _identity},
|
|
'session': {'call_id': '', 'application': 'm.call', 'scope': 'm.room'},
|
|
'sent_ts': DateTime.now().millisecondsSinceEpoch,
|
|
});
|
|
} catch (e) {
|
|
Logs().w('Could not share the call key', e);
|
|
for (final m in targets) {
|
|
_keySentTo.remove('${m.userId}:${m.deviceId}');
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Someone's media key, over an encrypted to-device message.
|
|
void _onToDevice(ToDeviceEvent event) {
|
|
if (event.type != keysType) return;
|
|
// Keys sent in the clear are refused, as the web client refuses them.
|
|
if (event.encryptedContent == null) return;
|
|
final content = event.content;
|
|
final keys = content.tryGetMap<String, Object?>('keys');
|
|
final member = content.tryGetMap<String, Object?>('member');
|
|
final key = keys?.tryGet<String>('key');
|
|
final index = keys?.tryGet<int>('index');
|
|
final device = member?.tryGet<String>('claimed_device_id');
|
|
if (key == null || index == null || device == null) return;
|
|
final identity = '${event.senderId}:$device';
|
|
Uint8List bytes;
|
|
try {
|
|
bytes = base64Decode(base64.normalize(key));
|
|
} catch (_) {
|
|
return;
|
|
}
|
|
final provider = _keyProvider;
|
|
if (provider == null || content['room_id'] != roomId) {
|
|
// Keys can come before we have joined: keep them for then.
|
|
_pendingKeys[identity] = (bytes, index);
|
|
return;
|
|
}
|
|
_setKey(provider, identity, bytes, index);
|
|
}
|
|
|
|
Future<void> _setKey(
|
|
lk.BaseKeyProvider provider,
|
|
String identity,
|
|
Uint8List key,
|
|
int index,
|
|
) async {
|
|
await provider.setRawKey(key, participantId: identity, keyIndex: index);
|
|
if (_keyProvider != provider || !_keyed.add(identity)) return;
|
|
// Video that came before its key was dropped, keyframe and all, and
|
|
// the decoder waits for another that may never come: black, while the
|
|
// audio plays on. Subscribing again makes the server ask for one.
|
|
final p = livekit?.getParticipantByIdentity(identity);
|
|
if (p is! lk.RemoteParticipant) return;
|
|
for (final pub in p.videoTrackPublications) {
|
|
if (!pub.subscribed) continue;
|
|
await pub.unsubscribe();
|
|
await pub.subscribe();
|
|
}
|
|
}
|
|
|
|
// --- Controls ---------------------------------------------------------------
|
|
|
|
bool get micOn => livekit?.localParticipant?.isMicrophoneEnabled() ?? false;
|
|
bool get cameraOn => livekit?.localParticipant?.isCameraEnabled() ?? false;
|
|
bool get screenSharing =>
|
|
livekit?.localParticipant?.isScreenShareEnabled() ?? false;
|
|
|
|
/// Screen sharing: Android only (iOS needs a broadcast extension).
|
|
bool get canShareScreen => Platform.isAndroid;
|
|
|
|
Future<void> toggleMic() async {
|
|
final p = livekit?.localParticipant;
|
|
if (p == null) return;
|
|
await p.setMicrophoneEnabled(!p.isMicrophoneEnabled());
|
|
notifyListeners();
|
|
}
|
|
|
|
Future<void> toggleCamera() async {
|
|
final p = livekit?.localParticipant;
|
|
if (p == null) return;
|
|
await p.setCameraEnabled(!p.isCameraEnabled());
|
|
notifyListeners();
|
|
}
|
|
|
|
/// Starts or stops sharing the screen. Android asks the user first, and
|
|
/// the call service must declare media projection before the capture
|
|
/// starts.
|
|
Future<void> toggleScreenShare() async {
|
|
final p = livekit?.localParticipant;
|
|
if (p == null) return;
|
|
if (p.isScreenShareEnabled()) {
|
|
await _stopScreenShare();
|
|
return;
|
|
}
|
|
if (!canShareScreen) return;
|
|
try {
|
|
if (!await Helper.requestCapturePermission()) return;
|
|
_sharingScreen = true;
|
|
if (!await _syncService()) {
|
|
_sharingScreen = false;
|
|
errors.add("Android didn't let the app share the screen.");
|
|
return;
|
|
}
|
|
final pub = await p.setScreenShareEnabled(true);
|
|
// Stopped from Android's own controls: unpublish it too. LiveKit
|
|
// listens to the track's end as well, so its handler is kept.
|
|
final track = pub?.track?.mediaStreamTrack;
|
|
if (track != null) {
|
|
final previous = track.onEnded;
|
|
track.onEnded = () {
|
|
previous?.call();
|
|
if (_sharingScreen) _stopScreenShare();
|
|
};
|
|
}
|
|
} catch (e) {
|
|
_sharingScreen = false;
|
|
errors.add("Can't share the screen: $e");
|
|
}
|
|
notifyListeners();
|
|
}
|
|
|
|
Future<void> _stopScreenShare() async {
|
|
_sharingScreen = false;
|
|
try {
|
|
await livekit?.localParticipant?.setScreenShareEnabled(false);
|
|
} catch (e) {
|
|
Logs().w('Could not stop sharing the screen', e);
|
|
}
|
|
notifyListeners();
|
|
}
|
|
|
|
/// Switches between the front and the back camera.
|
|
Future<void> flipCamera() async {
|
|
final track = livekit?.localParticipant?.videoTrackPublications
|
|
.where((t) => t.source == lk.TrackSource.camera)
|
|
.firstOrNull
|
|
?.track;
|
|
if (track is! lk.LocalVideoTrack) return;
|
|
final options = track.currentOptions;
|
|
if (options is! lk.CameraCaptureOptions) return;
|
|
await track.setCameraPosition(options.cameraPosition.switched());
|
|
notifyListeners();
|
|
}
|
|
|
|
// --- The call service ------------------------------------------------------
|
|
|
|
/// The microphone and camera have been asked for.
|
|
bool _mediaStarted = false;
|
|
|
|
/// Screen sharing is on, or about to be.
|
|
bool _sharingScreen = false;
|
|
|
|
/// What the service was last told, so it is told only of changes.
|
|
String? _serviceState;
|
|
|
|
@override
|
|
void notifyListeners() {
|
|
super.notifyListeners();
|
|
_syncService();
|
|
}
|
|
|
|
/// Starts, updates or stops the call service to match the call.
|
|
Future<bool> _syncService() async {
|
|
final rid = roomId;
|
|
final since = joinedAt;
|
|
if (rid == null || since == null || !_mediaStarted) {
|
|
if (_serviceState != null) {
|
|
_serviceState = null;
|
|
await CallService.stop();
|
|
}
|
|
return false;
|
|
}
|
|
final title = client.getRoomById(rid)?.getLocalizedDisplayname() ?? 'Call';
|
|
final camera = cameraOn;
|
|
final screen = _sharingScreen || screenSharing;
|
|
final state = '$rid|$title|$camera|$screen';
|
|
if (state == _serviceState) return true;
|
|
_serviceState = state;
|
|
final ok = await CallService.update(
|
|
title: title,
|
|
startedAt: since,
|
|
camera: camera,
|
|
screen: screen,
|
|
);
|
|
if (!ok) _serviceState = null;
|
|
return ok;
|
|
}
|
|
|
|
@override
|
|
void dispose() {
|
|
for (final s in _subs) {
|
|
s.cancel();
|
|
}
|
|
_expiryTimer?.cancel();
|
|
leave();
|
|
CallService.stop();
|
|
errors.close();
|
|
_openRequests.close();
|
|
super.dispose();
|
|
}
|
|
}
|
|
|
|
enum CallPhase { joining, connected, reconnecting }
|
|
|
|
class CallException implements Exception {
|
|
const CallException(this.message);
|
|
final String message;
|
|
@override
|
|
String toString() => message;
|
|
}
|
|
|
|
/// A call ringing for us.
|
|
class IncomingCall {
|
|
const IncomingCall({
|
|
required this.roomId,
|
|
required this.eventId,
|
|
required this.callerId,
|
|
required this.callerName,
|
|
required this.callerAvatar,
|
|
required this.roomName,
|
|
required this.video,
|
|
required this.expiresAt,
|
|
});
|
|
|
|
final String roomId;
|
|
|
|
/// The notification event, which a decline refers to.
|
|
final String eventId;
|
|
final String callerId;
|
|
final String callerName;
|
|
final Uri? callerAvatar;
|
|
final String roomName;
|
|
final bool video;
|
|
final DateTime expiresAt;
|
|
}
|
|
|
|
/// One device's membership of a room's call.
|
|
class CallMember {
|
|
CallMember({
|
|
required this.userId,
|
|
required this.deviceId,
|
|
required this.createdAt,
|
|
required this.expiresAt,
|
|
required this.transport,
|
|
});
|
|
|
|
final String userId;
|
|
final String deviceId;
|
|
final DateTime createdAt;
|
|
final DateTime expiresAt;
|
|
|
|
/// The LiveKit transport it publishes to.
|
|
final Map<String, Object?>? transport;
|
|
|
|
/// Reads a call member state event; null for an empty one (a leave) or
|
|
/// one that isn't a room call.
|
|
static CallMember? fromState(StrippedStateEvent state) {
|
|
final content = state.content;
|
|
if (content.isEmpty) return null;
|
|
if (content['application'] != 'm.call') return null;
|
|
final deviceId = content.tryGet<String>('device_id');
|
|
if (deviceId == null) return null;
|
|
final origin = state is Event
|
|
? state.originServerTs
|
|
: DateTime.fromMillisecondsSinceEpoch(0);
|
|
final created = content.tryGet<int>('created_ts');
|
|
final createdAt = created == null
|
|
? origin
|
|
: DateTime.fromMillisecondsSinceEpoch(created);
|
|
final expires = content.tryGet<int>('expires') ?? 4 * 60 * 60 * 1000;
|
|
final foci = content.tryGetList<Object?>('foci_preferred');
|
|
final transport = foci
|
|
?.whereType<Map>()
|
|
.where(
|
|
(f) => f['type'] == 'livekit' && f['livekit_service_url'] is String,
|
|
)
|
|
.map((f) => Map<String, Object?>.from(f))
|
|
.firstOrNull;
|
|
return CallMember(
|
|
userId: state.senderId,
|
|
deviceId: deviceId,
|
|
createdAt: createdAt,
|
|
expiresAt: createdAt.add(Duration(milliseconds: expires)),
|
|
transport: transport,
|
|
);
|
|
}
|
|
}
|