tajniak81andClaude Opus 5.5
12550c4078
- Calls keep going with the app in the background: a foreground service (CallService.kt, driven from lib/matrix/call_service.dart) declares the microphone, the camera while it's on and the screen while it's shared. Its notification (CallStyle) shows the call's length, opens the call and hangs up. Swiping the app away ends the call. - Screen sharing on Android: consent, then media projection on the service, then LiveKit publishes a screen-share track. A shared screen replaces the sharer's camera on their tile, shown whole and unmirrored; stopping it from Android's controls stops the share. - Read receipts per thread (MSC3771): an open thread sends receipts with thread_id, "Read" under replies comes from receipts in the thread, and threads with unread replies get a dot on "N replies" and in the list. Not tested on a device yet. README updated. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
972 lines
30 KiB
Dart
972 lines
30 KiB
Dart
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
import 'dart:math';
|
|
|
|
import 'package:flutter/foundation.dart';
|
|
import 'package:http/http.dart' as http;
|
|
import 'package:livekit_client/livekit_client.dart' as lk;
|
|
import 'package:matrix/matrix.dart';
|
|
|
|
import 'call_service.dart';
|
|
import 'notifications.dart';
|
|
import 'package:webrtc_interface/webrtc_interface.dart' as rtc;
|
|
import 'package:flutter_webrtc/flutter_webrtc.dart'
|
|
show Helper, frameCryptorFactory;
|
|
|
|
/// Voice and video calls: MatrixRTC (MSC4143) with a LiveKit SFU, the way
|
|
/// the web client and Element Call do them, so the three can share a call.
|
|
///
|
|
/// Who is in a room's call is room state: one
|
|
/// `org.matrix.msc3401.call.member` event per device, state key
|
|
/// `_@user:server_DEVICE_m.call`. Joining sends ours, after scheduling an
|
|
/// empty one with the homeserver (a delayed event, MSC4140) that is pushed
|
|
/// back every few seconds while the call lasts: if the app dies or loses
|
|
/// its connection, the server sends it and the device drops out.
|
|
///
|
|
/// The media goes through LiveKit, whose token the LiveKit JWT service
|
|
/// hands out for an OpenID token from the homeserver. In an encrypted room
|
|
/// every member makes its own media key and sends it to the others over
|
|
/// encrypted to-device messages (`io.element.call.encryption_keys`); LiveKit
|
|
/// encrypts each frame with it.
|
|
///
|
|
/// Starting a call in a room of two rings the other person
|
|
/// (`org.matrix.msc4075.rtc.notification`); declining answers it
|
|
/// (`org.matrix.msc4310.rtc.decline`).
|
|
///
|
|
/// While this device is in a call, a foreground service on Android
|
|
/// ([CallService]) keeps it going with the app in the background, and lets
|
|
/// it share the screen.
|
|
///
|
|
/// One call at a time.
|
|
class Calls extends ChangeNotifier {
|
|
Calls(this.client) {
|
|
_subs
|
|
..add(client.onSync.stream.listen((_) => _refresh()))
|
|
..add(client.onTimelineEvent.stream.listen(_onTimelineEvent))
|
|
..add(client.onToDeviceEvent.stream.listen(_onToDevice));
|
|
CallService.listen(onHangUp: leave, onOpen: () => _openRequests.add(null));
|
|
_expiryTimer = Timer.periodic(
|
|
const Duration(seconds: 30),
|
|
(_) => _refresh(),
|
|
);
|
|
_refresh();
|
|
}
|
|
|
|
final Client client;
|
|
final _subs = <StreamSubscription>[];
|
|
Timer? _expiryTimer;
|
|
|
|
static const memberType = 'org.matrix.msc3401.call.member';
|
|
static const notificationType = 'org.matrix.msc4075.rtc.notification';
|
|
static const declineType = 'org.matrix.msc4310.rtc.decline';
|
|
static const keysType = 'io.element.call.encryption_keys';
|
|
|
|
/// How long a call rings, unless the notification says less.
|
|
static const ringLifetime = Duration(seconds: 60);
|
|
|
|
/// The most a received notification may ring for, whatever it says.
|
|
static const maxRing = Duration(seconds: 120);
|
|
|
|
/// A membership lasts this long unless renewed; calls are shorter.
|
|
static const membershipExpiry = Duration(hours: 4);
|
|
|
|
static const _delayedLeave = Duration(seconds: 8);
|
|
static const _delayedLeaveRestart = Duration(seconds: 5);
|
|
|
|
// --- What the screens show --------------------------------------------
|
|
|
|
/// The room whose call this device is in.
|
|
String? roomId;
|
|
CallPhase? phase;
|
|
bool video = false;
|
|
bool encrypted = false;
|
|
lk.Room? livekit;
|
|
|
|
/// We rang someone and they haven't come yet.
|
|
bool ringingOut = false;
|
|
|
|
/// A call ringing for us.
|
|
IncomingCall? incoming;
|
|
|
|
/// When we joined, for the call's length.
|
|
DateTime? joinedAt;
|
|
|
|
/// How many people (not devices) are in each room's call.
|
|
final active = <String, int>{};
|
|
|
|
/// Errors to show once, then cleared.
|
|
final errors = StreamController<String>.broadcast();
|
|
|
|
final _openRequests = StreamController<void>.broadcast();
|
|
|
|
/// The ongoing call notification was tapped: show the call.
|
|
Stream<void> get openRequests => _openRequests.stream;
|
|
|
|
bool get inCall => roomId != null;
|
|
|
|
int inCallCount(String roomId) => active[roomId] ?? 0;
|
|
|
|
// --- Memberships ------------------------------------------------------
|
|
|
|
/// The live memberships of [room]'s call, one per device.
|
|
List<CallMember> memberships(Room room) {
|
|
final events = room.states[memberType]?.values ?? const [];
|
|
final now = DateTime.now();
|
|
final out = <CallMember>[];
|
|
for (final e in events) {
|
|
final m = CallMember.fromState(e);
|
|
if (m != null && m.expiresAt.isAfter(now)) out.add(m);
|
|
}
|
|
out.sort((a, b) => a.createdAt.compareTo(b.createdAt));
|
|
return out;
|
|
}
|
|
|
|
void _refresh() {
|
|
var changed = false;
|
|
final seen = <String>{};
|
|
for (final room in client.rooms) {
|
|
if (room.membership != Membership.join) continue;
|
|
final users = memberships(room).map((m) => m.userId).toSet();
|
|
final n = users.length;
|
|
if (n > 0) {
|
|
seen.add(room.id);
|
|
if (active[room.id] != n) {
|
|
active[room.id] = n;
|
|
changed = true;
|
|
}
|
|
}
|
|
}
|
|
for (final id in active.keys.toList()) {
|
|
if (!seen.contains(id)) {
|
|
active.remove(id);
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
// The caller gave up, or someone else answered for us.
|
|
final ring = incoming;
|
|
if (ring != null && inCallCount(ring.roomId) == 0) {
|
|
_clearIncoming();
|
|
changed = true;
|
|
}
|
|
final rid = roomId;
|
|
if (rid != null) {
|
|
final room = client.getRoomById(rid);
|
|
if (room != null) {
|
|
final members = memberships(room);
|
|
// Someone came: stop ringing them.
|
|
if (ringingOut && members.map((m) => m.userId).toSet().length > 1) {
|
|
ringingOut = false;
|
|
changed = true;
|
|
}
|
|
if (encrypted) _shareKeyWithNewDevices(room, members);
|
|
}
|
|
}
|
|
if (changed) notifyListeners();
|
|
}
|
|
|
|
// --- Ringing ------------------------------------------------------------
|
|
|
|
final _seenEvents = <String>{};
|
|
|
|
void _onTimelineEvent(Event event) {
|
|
if (event.type != notificationType && event.type != declineType) return;
|
|
if (!_seenEvents.add(event.eventId)) return;
|
|
final me = client.userID;
|
|
final room = event.room;
|
|
|
|
if (event.type == notificationType) {
|
|
if (event.senderId == me || roomId == room.id || inCall) return;
|
|
final mentions = event.content.tryGetMap<String, Object?>('m.mentions');
|
|
final userIds = mentions?.tryGetList<String>('user_ids') ?? const [];
|
|
if (mentions?['room'] != true && !userIds.contains(me)) return;
|
|
// A ring for a call of two; in a bigger room the banner says enough.
|
|
if (event.content['notification_type'] != 'ring' &&
|
|
_otherOfTwo(room) == null) {
|
|
return;
|
|
}
|
|
final senderTs = event.content.tryGet<int>('sender_ts');
|
|
final sentAt = senderTs == null
|
|
? event.originServerTs
|
|
: DateTime.fromMillisecondsSinceEpoch(
|
|
min(senderTs, event.originServerTs.millisecondsSinceEpoch),
|
|
);
|
|
final lifetimeMs = min(
|
|
event.content.tryGet<int>('lifetime') ?? ringLifetime.inMilliseconds,
|
|
maxRing.inMilliseconds,
|
|
);
|
|
final expiresAt = sentAt.add(Duration(milliseconds: lifetimeMs));
|
|
if (!expiresAt.isAfter(DateTime.now())) return;
|
|
final caller = event.senderFromMemoryOrFallback;
|
|
incoming = IncomingCall(
|
|
roomId: room.id,
|
|
eventId: event.eventId,
|
|
callerId: event.senderId,
|
|
callerName: caller.calcDisplayname(),
|
|
callerAvatar: caller.avatarUrl,
|
|
roomName: room.getLocalizedDisplayname(),
|
|
video: event.content['m.call.intent'] != 'audio',
|
|
expiresAt: expiresAt,
|
|
);
|
|
_ringTimer?.cancel();
|
|
_ringTimer = Timer(expiresAt.difference(DateTime.now()), () {
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
});
|
|
Notifications.ring(
|
|
roomId: room.id,
|
|
caller: incoming!.callerName,
|
|
video: incoming!.video,
|
|
);
|
|
notifyListeners();
|
|
return;
|
|
}
|
|
|
|
// A decline.
|
|
final target = event.relationshipEventId;
|
|
if (target == null) return;
|
|
if (event.senderId == me && incoming?.eventId == target) {
|
|
// Declined on another of our devices.
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
} else if (target == _sentNotification && ringingOut && room.id == roomId) {
|
|
errors.add(
|
|
'${event.senderFromMemoryOrFallback.calcDisplayname()} declined the call.',
|
|
);
|
|
leave();
|
|
}
|
|
}
|
|
|
|
Timer? _ringTimer;
|
|
|
|
void _clearIncoming() {
|
|
_ringTimer?.cancel();
|
|
if (incoming != null) Notifications.stopRinging();
|
|
incoming = null;
|
|
}
|
|
|
|
/// Answers the call ringing for us.
|
|
Future<void> answer({bool withVideo = true}) async {
|
|
final call = incoming;
|
|
if (call == null) return;
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
await join(call.roomId, video: withVideo && call.video);
|
|
}
|
|
|
|
/// Declines the call ringing for us.
|
|
Future<void> decline() async {
|
|
final call = incoming;
|
|
if (call == null) return;
|
|
_clearIncoming();
|
|
notifyListeners();
|
|
try {
|
|
await client.getRoomById(call.roomId)?.sendEvent({
|
|
'm.relates_to': {
|
|
'rel_type': RelationshipTypes.reference,
|
|
'event_id': call.eventId,
|
|
},
|
|
}, type: declineType);
|
|
} catch (e) {
|
|
Logs().w('Could not decline the call', e);
|
|
}
|
|
}
|
|
|
|
/// The other person in a room of two.
|
|
String? _otherOfTwo(Room room) {
|
|
if ((room.summary.mJoinedMemberCount ?? room.getParticipants().length) !=
|
|
2) {
|
|
return null;
|
|
}
|
|
return room
|
|
.getParticipants([Membership.join])
|
|
.map((u) => u.id)
|
|
.where((id) => id != client.userID)
|
|
.firstOrNull;
|
|
}
|
|
|
|
// --- Joining ------------------------------------------------------------
|
|
|
|
String? _sentNotification;
|
|
String? _delayId;
|
|
Timer? _delayTimer;
|
|
Timer? _ringOutTimer;
|
|
lk.BaseKeyProvider? _keyProvider;
|
|
lk.EventsListener<lk.RoomEvent>? _lkListener;
|
|
|
|
/// Our media key, its index, and who has it.
|
|
Uint8List? _key;
|
|
final _keyIndex = 0;
|
|
final _keySentTo = <String>{};
|
|
|
|
/// Keys that came before LiveKit was ready, by identity.
|
|
final _pendingKeys = <String, (Uint8List, int)>{};
|
|
|
|
String get _deviceId => client.deviceID!;
|
|
String get _identity => '${client.userID}:$_deviceId';
|
|
|
|
String _stateKey(Room room) {
|
|
final key = '${client.userID}_${_deviceId}_m.call';
|
|
// Room versions with MSC3757/3779 own-state keys take it bare.
|
|
final version = room
|
|
.getState(EventTypes.RoomCreate)
|
|
?.content
|
|
.tryGet<String>('room_version');
|
|
return version != null &&
|
|
RegExp(r'^org\.matrix\.msc(3757|3779)\b').hasMatch(version)
|
|
? key
|
|
: '_$key';
|
|
}
|
|
|
|
/// Joins the call in [roomId], or starts one. Starting a call in a room
|
|
/// of two rings the other person.
|
|
Future<void> join(String roomId, {required bool video}) async {
|
|
if (this.roomId == roomId) return;
|
|
if (inCall) await leave();
|
|
final room = client.getRoomById(roomId);
|
|
if (room == null) return;
|
|
if (incoming?.roomId == roomId) _clearIncoming();
|
|
|
|
this.roomId = roomId;
|
|
this.video = video;
|
|
phase = CallPhase.joining;
|
|
encrypted = room.encrypted;
|
|
_sentNotification = null;
|
|
_keySentTo.clear();
|
|
notifyListeners();
|
|
|
|
final others = memberships(room);
|
|
final ringing = others.isEmpty && _otherOfTwo(room) != null;
|
|
try {
|
|
final transport = await _chooseTransport(others);
|
|
final (url, jwt) = await _livekitToken(transport, roomId);
|
|
if (this.roomId != roomId) return;
|
|
|
|
if (encrypted) {
|
|
_keyProvider = await _makeKeyProvider();
|
|
_key = _randomKey();
|
|
await _keyProvider!.setRawKey(
|
|
_key!,
|
|
participantId: _identity,
|
|
keyIndex: _keyIndex,
|
|
);
|
|
for (final e in _pendingKeys.entries) {
|
|
await _keyProvider!.setRawKey(
|
|
e.value.$1,
|
|
participantId: e.key,
|
|
keyIndex: e.value.$2,
|
|
);
|
|
}
|
|
_pendingKeys.clear();
|
|
}
|
|
|
|
final lkRoom = lk.Room(
|
|
roomOptions: lk.RoomOptions(
|
|
adaptiveStream: true,
|
|
dynacast: true,
|
|
e2eeOptions: _keyProvider == null
|
|
? null
|
|
: lk.E2EEOptions(keyProvider: _keyProvider!),
|
|
),
|
|
);
|
|
livekit = lkRoom;
|
|
lkRoom.addListener(_onLivekit);
|
|
_lkListener = lkRoom.createListener()
|
|
..on<lk.RoomReconnectingEvent>((_) {
|
|
phase = CallPhase.reconnecting;
|
|
notifyListeners();
|
|
})
|
|
..on<lk.RoomReconnectedEvent>((_) {
|
|
phase = CallPhase.connected;
|
|
notifyListeners();
|
|
})
|
|
..on<lk.RoomDisconnectedEvent>((event) {
|
|
if (livekit != lkRoom) return;
|
|
if (event.reason != lk.DisconnectReason.clientInitiated) {
|
|
errors.add('You were disconnected from the call.');
|
|
}
|
|
leave();
|
|
});
|
|
|
|
// The Matrix side: a delayed leave first, then ourselves.
|
|
await _scheduleDelayedLeave(room);
|
|
final membershipEventId = await client.setRoomStateWithKey(
|
|
roomId,
|
|
memberType,
|
|
_stateKey(room),
|
|
_myMembership(transport, video),
|
|
);
|
|
if (encrypted) _shareKeyWithNewDevices(room, memberships(room));
|
|
if (ringing || others.isEmpty) {
|
|
_sendNotification(room, membershipEventId, ring: ringing, video: video);
|
|
}
|
|
if (ringing) {
|
|
ringingOut = true;
|
|
_ringOutTimer = Timer(ringLifetime, () {
|
|
if (!ringingOut || this.roomId != roomId) return;
|
|
errors.add('No answer.');
|
|
leave();
|
|
});
|
|
}
|
|
|
|
await lkRoom.connect(url, jwt);
|
|
if (this.roomId != roomId) {
|
|
await lkRoom.disconnect();
|
|
return;
|
|
}
|
|
if (_keyProvider != null) await lkRoom.e2eeManager?.setEnabled(true);
|
|
phase = CallPhase.connected;
|
|
joinedAt = DateTime.now();
|
|
notifyListeners();
|
|
|
|
try {
|
|
await lkRoom.localParticipant?.setMicrophoneEnabled(true);
|
|
} catch (e) {
|
|
errors.add("Can't use the microphone: $e");
|
|
}
|
|
if (video) {
|
|
try {
|
|
await lkRoom.localParticipant?.setCameraEnabled(true);
|
|
} catch (e) {
|
|
errors.add("Can't use the camera: $e");
|
|
}
|
|
}
|
|
// Now that the microphone and camera are asked for, the service can
|
|
// declare them.
|
|
_mediaStarted = true;
|
|
notifyListeners();
|
|
} catch (e, s) {
|
|
Logs().e('Joining the call failed', e, s);
|
|
errors.add(e is CallException ? e.message : 'The call failed: $e');
|
|
await leave();
|
|
}
|
|
}
|
|
|
|
void _onLivekit() => notifyListeners();
|
|
|
|
/// Leaves the call this device is in.
|
|
Future<void> leave() async {
|
|
final rid = roomId;
|
|
final lkRoom = livekit;
|
|
roomId = null;
|
|
livekit = null;
|
|
phase = null;
|
|
joinedAt = null;
|
|
ringingOut = false;
|
|
_mediaStarted = false;
|
|
_sharingScreen = false;
|
|
_ringOutTimer?.cancel();
|
|
_delayTimer?.cancel();
|
|
_keyProvider = null;
|
|
_key = null;
|
|
_keySentTo.clear();
|
|
_pendingKeys.clear();
|
|
final delayId = _delayId;
|
|
_delayId = null;
|
|
notifyListeners();
|
|
|
|
lkRoom?.removeListener(_onLivekit);
|
|
await _lkListener?.dispose();
|
|
_lkListener = null;
|
|
try {
|
|
await lkRoom?.disconnect();
|
|
} catch (_) {}
|
|
if (rid != null) {
|
|
final room = client.getRoomById(rid);
|
|
if (room != null) {
|
|
try {
|
|
await client.setRoomStateWithKey(
|
|
rid,
|
|
memberType,
|
|
_stateKey(room),
|
|
{},
|
|
);
|
|
} catch (e) {
|
|
Logs().w('Could not send the leave; the delayed one will', e);
|
|
}
|
|
}
|
|
}
|
|
if (delayId != null) {
|
|
try {
|
|
await _delayedEventAction(delayId, 'cancel');
|
|
} catch (_) {}
|
|
}
|
|
_refresh();
|
|
}
|
|
|
|
Map<String, Object?> _myMembership(
|
|
Map<String, Object?> transport,
|
|
bool video,
|
|
) => {
|
|
'application': 'm.call',
|
|
'call_id': '',
|
|
'scope': 'm.room',
|
|
'device_id': _deviceId,
|
|
'membershipID': _identity,
|
|
'expires': membershipExpiry.inMilliseconds,
|
|
'm.call.intent': video ? 'video' : 'audio',
|
|
'focus_active': {'type': 'livekit', 'focus_selection': 'multi_sfu'},
|
|
'foci_preferred': [transport],
|
|
};
|
|
|
|
void _sendNotification(
|
|
Room room,
|
|
String membershipEventId, {
|
|
required bool ring,
|
|
required bool video,
|
|
}) {
|
|
room
|
|
.sendEvent({
|
|
'slot_id': 'm.call#ROOM',
|
|
'm.mentions': {'user_ids': <String>[], 'room': true},
|
|
'notification_type': ring ? 'ring' : 'notification',
|
|
'm.relates_to': {
|
|
'event_id': membershipEventId,
|
|
'rel_type': RelationshipTypes.reference,
|
|
},
|
|
'sender_ts': DateTime.now().millisecondsSinceEpoch,
|
|
'lifetime': ringLifetime.inMilliseconds,
|
|
'msc4354_sticky_key': 'm.call#ROOM',
|
|
'm.call.intent': video ? 'video' : 'audio',
|
|
}, type: notificationType)
|
|
.then((id) => _sentNotification = id)
|
|
.catchError((Object e) {
|
|
Logs().w('Could not send the call notification', e);
|
|
return null;
|
|
});
|
|
}
|
|
|
|
// --- Delayed leave (MSC4140) ----------------------------------------------
|
|
|
|
Future<void> _scheduleDelayedLeave(Room room) async {
|
|
try {
|
|
final response = await client.request(
|
|
RequestType.PUT,
|
|
'/client/v3/rooms/${Uri.encodeComponent(room.id)}/state/'
|
|
'${Uri.encodeComponent(memberType)}/${Uri.encodeComponent(_stateKey(room))}',
|
|
query: {'org.matrix.msc4140.delay': '${_delayedLeave.inMilliseconds}'},
|
|
data: <String, Object?>{},
|
|
);
|
|
final id = response['delay_id'];
|
|
if (id is! String) return;
|
|
_delayId = id;
|
|
_delayTimer = Timer.periodic(_delayedLeaveRestart, (_) {
|
|
final current = _delayId;
|
|
if (current == null) return;
|
|
_delayedEventAction(current, 'restart').catchError((Object e) {
|
|
Logs().w('Could not push back the delayed leave', e);
|
|
});
|
|
});
|
|
} catch (e) {
|
|
// Without it, a crash leaves us in the call until our membership
|
|
// expires; the call itself still works.
|
|
Logs().w('The server has no delayed events', e);
|
|
}
|
|
}
|
|
|
|
Future<void> _delayedEventAction(String id, String action) => client.request(
|
|
RequestType.POST,
|
|
'/client/unstable/org.matrix.msc4140/delayed_events/'
|
|
'${Uri.encodeComponent(id)}/$action',
|
|
data: <String, Object?>{},
|
|
);
|
|
|
|
// --- LiveKit --------------------------------------------------------------
|
|
|
|
/// The LiveKit JWT service: the one the call already uses, else this
|
|
/// server's.
|
|
Future<Map<String, Object?>> _chooseTransport(List<CallMember> others) async {
|
|
for (final m in others) {
|
|
final t = m.transport;
|
|
if (t != null) return {...t, 'livekit_alias': roomId};
|
|
}
|
|
List? foci;
|
|
try {
|
|
final response = await client.request(
|
|
RequestType.GET,
|
|
'/client/unstable/org.matrix.msc4143/rtc/transports',
|
|
);
|
|
foci = response['rtc_transports'] as List?;
|
|
} catch (_) {
|
|
try {
|
|
final wellKnown = await http.get(
|
|
client.homeserver!.resolve('/.well-known/matrix/client'),
|
|
);
|
|
final json = jsonDecode(wellKnown.body);
|
|
foci = json is Map
|
|
? json['org.matrix.msc4143.rtc_foci'] as List?
|
|
: null;
|
|
} catch (_) {}
|
|
}
|
|
final focus = foci
|
|
?.whereType<Map>()
|
|
.where(
|
|
(f) => f['type'] == 'livekit' && f['livekit_service_url'] is String,
|
|
)
|
|
.firstOrNull;
|
|
if (focus == null) {
|
|
throw const CallException("This server doesn't offer calls.");
|
|
}
|
|
return {
|
|
'type': 'livekit',
|
|
'livekit_service_url': focus['livekit_service_url'],
|
|
'livekit_alias': roomId,
|
|
};
|
|
}
|
|
|
|
/// A LiveKit URL and token from the JWT service, for an OpenID token.
|
|
Future<(String, String)> _livekitToken(
|
|
Map<String, Object?> transport,
|
|
String roomId,
|
|
) async {
|
|
final openid = await client.requestOpenIdToken(client.userID!, {});
|
|
final service = (transport['livekit_service_url'] as String).replaceFirst(
|
|
RegExp(r'/$'),
|
|
'',
|
|
);
|
|
final response = await http.post(
|
|
Uri.parse('$service/sfu/get'),
|
|
headers: {'Content-Type': 'application/json'},
|
|
body: jsonEncode({
|
|
'room': roomId,
|
|
'openid_token': openid.toJson(),
|
|
'device_id': _deviceId,
|
|
}),
|
|
);
|
|
if (response.statusCode != 200) {
|
|
throw CallException(
|
|
'The call service refused to let you in (${response.statusCode}).',
|
|
);
|
|
}
|
|
final body = jsonDecode(response.body);
|
|
if (body is! Map || body['url'] is! String || body['jwt'] is! String) {
|
|
throw const CallException('The call service gave an unusable answer.');
|
|
}
|
|
return (body['url'] as String, body['jwt'] as String);
|
|
}
|
|
|
|
/// Per-participant keys, no ratcheting, keys derived with HKDF: the
|
|
/// settings of the web client and Element Call, whose livekit-client
|
|
/// imports keys as HKDF material.
|
|
Future<lk.BaseKeyProvider> _makeKeyProvider() async {
|
|
final options = rtc.KeyProviderOptions(
|
|
sharedKey: false,
|
|
ratchetSalt: Uint8List.fromList(utf8.encode('LKFrameEncryptionKey')),
|
|
ratchetWindowSize: 0,
|
|
uncryptedMagicBytes: Uint8List.fromList(utf8.encode('LK-ROCKS')),
|
|
failureTolerance: -1,
|
|
keyRingSize: 256,
|
|
keyDerivationAlgorithm: rtc.KeyDerivationAlgorithm.kHKDF,
|
|
);
|
|
final provider = await frameCryptorFactory.createDefaultKeyProvider(
|
|
options,
|
|
);
|
|
return lk.BaseKeyProvider(provider, options);
|
|
}
|
|
|
|
// --- Media keys -----------------------------------------------------------
|
|
|
|
Uint8List _randomKey() {
|
|
final random = Random.secure();
|
|
return Uint8List.fromList(List.generate(16, (_) => random.nextInt(256)));
|
|
}
|
|
|
|
/// Sends our media key to every device in the call that hasn't had it.
|
|
Future<void> _shareKeyWithNewDevices(
|
|
Room room,
|
|
List<CallMember> members,
|
|
) async {
|
|
final key = _key;
|
|
if (key == null) return;
|
|
final targets = members
|
|
.where(
|
|
(m) =>
|
|
!(m.userId == client.userID && m.deviceId == _deviceId) &&
|
|
!_keySentTo.contains('${m.userId}:${m.deviceId}'),
|
|
)
|
|
.toList();
|
|
if (targets.isEmpty) return;
|
|
for (final m in targets) {
|
|
_keySentTo.add('${m.userId}:${m.deviceId}');
|
|
}
|
|
try {
|
|
final users = targets.map((m) => m.userId).toSet();
|
|
if (users.any((u) => client.userDeviceKeys[u] == null)) {
|
|
await client.updateUserDeviceKeys(additionalUsers: users);
|
|
}
|
|
final devices = [
|
|
for (final m in targets)
|
|
?client.userDeviceKeys[m.userId]?.deviceKeys[m.deviceId],
|
|
];
|
|
await client.sendToDeviceEncrypted(devices, keysType, {
|
|
'keys': {'index': _keyIndex, 'key': base64Encode(key)},
|
|
'room_id': room.id,
|
|
'member': {'claimed_device_id': _deviceId, 'id': _identity},
|
|
'session': {'call_id': '', 'application': 'm.call', 'scope': 'm.room'},
|
|
'sent_ts': DateTime.now().millisecondsSinceEpoch,
|
|
});
|
|
} catch (e) {
|
|
Logs().w('Could not share the call key', e);
|
|
for (final m in targets) {
|
|
_keySentTo.remove('${m.userId}:${m.deviceId}');
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Someone's media key, over an encrypted to-device message.
|
|
void _onToDevice(ToDeviceEvent event) {
|
|
if (event.type != keysType) return;
|
|
// Keys sent in the clear are refused, as the web client refuses them.
|
|
if (event.encryptedContent == null) return;
|
|
final content = event.content;
|
|
final keys = content.tryGetMap<String, Object?>('keys');
|
|
final member = content.tryGetMap<String, Object?>('member');
|
|
final key = keys?.tryGet<String>('key');
|
|
final index = keys?.tryGet<int>('index');
|
|
final device = member?.tryGet<String>('claimed_device_id');
|
|
if (key == null || index == null || device == null) return;
|
|
final identity = '${event.senderId}:$device';
|
|
Uint8List bytes;
|
|
try {
|
|
bytes = base64Decode(base64.normalize(key));
|
|
} catch (_) {
|
|
return;
|
|
}
|
|
final provider = _keyProvider;
|
|
if (provider == null || content['room_id'] != roomId) {
|
|
// Keys can come before we have joined: keep them for then.
|
|
_pendingKeys[identity] = (bytes, index);
|
|
return;
|
|
}
|
|
provider.setRawKey(bytes, participantId: identity, keyIndex: index);
|
|
}
|
|
|
|
// --- Controls ---------------------------------------------------------------
|
|
|
|
bool get micOn => livekit?.localParticipant?.isMicrophoneEnabled() ?? false;
|
|
bool get cameraOn => livekit?.localParticipant?.isCameraEnabled() ?? false;
|
|
bool get screenSharing =>
|
|
livekit?.localParticipant?.isScreenShareEnabled() ?? false;
|
|
|
|
/// Screen sharing: Android only (iOS needs a broadcast extension).
|
|
bool get canShareScreen => Platform.isAndroid;
|
|
|
|
Future<void> toggleMic() async {
|
|
final p = livekit?.localParticipant;
|
|
if (p == null) return;
|
|
await p.setMicrophoneEnabled(!p.isMicrophoneEnabled());
|
|
notifyListeners();
|
|
}
|
|
|
|
Future<void> toggleCamera() async {
|
|
final p = livekit?.localParticipant;
|
|
if (p == null) return;
|
|
await p.setCameraEnabled(!p.isCameraEnabled());
|
|
notifyListeners();
|
|
}
|
|
|
|
/// Starts or stops sharing the screen. Android asks the user first, and
|
|
/// the call service must declare media projection before the capture
|
|
/// starts.
|
|
Future<void> toggleScreenShare() async {
|
|
final p = livekit?.localParticipant;
|
|
if (p == null) return;
|
|
if (p.isScreenShareEnabled()) {
|
|
await _stopScreenShare();
|
|
return;
|
|
}
|
|
if (!canShareScreen) return;
|
|
try {
|
|
if (!await Helper.requestCapturePermission()) return;
|
|
_sharingScreen = true;
|
|
if (!await _syncService()) {
|
|
_sharingScreen = false;
|
|
errors.add("Android didn't let the app share the screen.");
|
|
return;
|
|
}
|
|
final pub = await p.setScreenShareEnabled(true);
|
|
// Stopped from Android's own controls: unpublish it too. LiveKit
|
|
// listens to the track's end as well, so its handler is kept.
|
|
final track = pub?.track?.mediaStreamTrack;
|
|
if (track != null) {
|
|
final previous = track.onEnded;
|
|
track.onEnded = () {
|
|
previous?.call();
|
|
if (_sharingScreen) _stopScreenShare();
|
|
};
|
|
}
|
|
} catch (e) {
|
|
_sharingScreen = false;
|
|
errors.add("Can't share the screen: $e");
|
|
}
|
|
notifyListeners();
|
|
}
|
|
|
|
Future<void> _stopScreenShare() async {
|
|
_sharingScreen = false;
|
|
try {
|
|
await livekit?.localParticipant?.setScreenShareEnabled(false);
|
|
} catch (e) {
|
|
Logs().w('Could not stop sharing the screen', e);
|
|
}
|
|
notifyListeners();
|
|
}
|
|
|
|
/// Switches between the front and the back camera.
|
|
Future<void> flipCamera() async {
|
|
final track = livekit?.localParticipant?.videoTrackPublications
|
|
.where((t) => t.source == lk.TrackSource.camera)
|
|
.firstOrNull
|
|
?.track;
|
|
if (track is! lk.LocalVideoTrack) return;
|
|
final options = track.currentOptions;
|
|
if (options is! lk.CameraCaptureOptions) return;
|
|
await track.setCameraPosition(options.cameraPosition.switched());
|
|
notifyListeners();
|
|
}
|
|
|
|
// --- The call service ------------------------------------------------------
|
|
|
|
/// The microphone and camera have been asked for.
|
|
bool _mediaStarted = false;
|
|
|
|
/// Screen sharing is on, or about to be.
|
|
bool _sharingScreen = false;
|
|
|
|
/// What the service was last told, so it is told only of changes.
|
|
String? _serviceState;
|
|
|
|
@override
|
|
void notifyListeners() {
|
|
super.notifyListeners();
|
|
_syncService();
|
|
}
|
|
|
|
/// Starts, updates or stops the call service to match the call.
|
|
Future<bool> _syncService() async {
|
|
final rid = roomId;
|
|
final since = joinedAt;
|
|
if (rid == null || since == null || !_mediaStarted) {
|
|
if (_serviceState != null) {
|
|
_serviceState = null;
|
|
await CallService.stop();
|
|
}
|
|
return false;
|
|
}
|
|
final title = client.getRoomById(rid)?.getLocalizedDisplayname() ?? 'Call';
|
|
final camera = cameraOn;
|
|
final screen = _sharingScreen || screenSharing;
|
|
final state = '$rid|$title|$camera|$screen';
|
|
if (state == _serviceState) return true;
|
|
_serviceState = state;
|
|
final ok = await CallService.update(
|
|
title: title,
|
|
startedAt: since,
|
|
camera: camera,
|
|
screen: screen,
|
|
);
|
|
if (!ok) _serviceState = null;
|
|
return ok;
|
|
}
|
|
|
|
@override
|
|
void dispose() {
|
|
for (final s in _subs) {
|
|
s.cancel();
|
|
}
|
|
_expiryTimer?.cancel();
|
|
leave();
|
|
CallService.stop();
|
|
errors.close();
|
|
_openRequests.close();
|
|
super.dispose();
|
|
}
|
|
}
|
|
|
|
enum CallPhase { joining, connected, reconnecting }
|
|
|
|
class CallException implements Exception {
|
|
const CallException(this.message);
|
|
final String message;
|
|
@override
|
|
String toString() => message;
|
|
}
|
|
|
|
/// A call ringing for us.
|
|
class IncomingCall {
|
|
const IncomingCall({
|
|
required this.roomId,
|
|
required this.eventId,
|
|
required this.callerId,
|
|
required this.callerName,
|
|
required this.callerAvatar,
|
|
required this.roomName,
|
|
required this.video,
|
|
required this.expiresAt,
|
|
});
|
|
|
|
final String roomId;
|
|
|
|
/// The notification event, which a decline refers to.
|
|
final String eventId;
|
|
final String callerId;
|
|
final String callerName;
|
|
final Uri? callerAvatar;
|
|
final String roomName;
|
|
final bool video;
|
|
final DateTime expiresAt;
|
|
}
|
|
|
|
/// One device's membership of a room's call.
|
|
class CallMember {
|
|
CallMember({
|
|
required this.userId,
|
|
required this.deviceId,
|
|
required this.createdAt,
|
|
required this.expiresAt,
|
|
required this.transport,
|
|
});
|
|
|
|
final String userId;
|
|
final String deviceId;
|
|
final DateTime createdAt;
|
|
final DateTime expiresAt;
|
|
|
|
/// The LiveKit transport it publishes to.
|
|
final Map<String, Object?>? transport;
|
|
|
|
/// Reads a call member state event; null for an empty one (a leave) or
|
|
/// one that isn't a room call.
|
|
static CallMember? fromState(StrippedStateEvent state) {
|
|
final content = state.content;
|
|
if (content.isEmpty) return null;
|
|
if (content['application'] != 'm.call') return null;
|
|
final deviceId = content.tryGet<String>('device_id');
|
|
if (deviceId == null) return null;
|
|
final origin = state is Event
|
|
? state.originServerTs
|
|
: DateTime.fromMillisecondsSinceEpoch(0);
|
|
final created = content.tryGet<int>('created_ts');
|
|
final createdAt = created == null
|
|
? origin
|
|
: DateTime.fromMillisecondsSinceEpoch(created);
|
|
final expires = content.tryGet<int>('expires') ?? 4 * 60 * 60 * 1000;
|
|
final foci = content.tryGetList<Object?>('foci_preferred');
|
|
final transport = foci
|
|
?.whereType<Map>()
|
|
.where(
|
|
(f) => f['type'] == 'livekit' && f['livekit_service_url'] is String,
|
|
)
|
|
.map((f) => Map<String, Object?>.from(f))
|
|
.firstOrNull;
|
|
return CallMember(
|
|
userId: state.senderId,
|
|
deviceId: deviceId,
|
|
createdAt: createdAt,
|
|
expiresAt: createdAt.add(Duration(milliseconds: expires)),
|
|
transport: transport,
|
|
);
|
|
}
|
|
}
|