diff --git a/apps/mobile/README.md b/apps/mobile/README.md index 12f1793..fca88ba 100644 --- a/apps/mobile/README.md +++ b/apps/mobile/README.md @@ -39,6 +39,10 @@ radii), the Outfit and Figtree fonts, the Warp symbol. - Threads: *Reply in thread* opens the thread, its root and replies, with a composer that sends into it. Replies stay out of the main timeline; the root shows "N replies". Room info lists a room's threads. + Read receipts per thread (MSC3771): an open thread marks itself read + with a receipt for the thread alone, "Read" under your replies comes + from the others' receipts in the thread, and threads with replies you + haven't read get a dot (on "N replies" and in the thread list). - Spaces: the room list's title picks a space, which narrows the list to its rooms (and its subspaces'). A space's browser (`/hierarchy`) shows the rooms you're in and the ones you can join; admins add existing or @@ -67,9 +71,9 @@ radii), the Outfit and Figtree fonts, the Warp symbol. - Settings: display name, avatar, theme (system, light, dark). **Not yet:** a per-message trust shield like the web client's, screen -sharing in calls, calls that keep going with the app in the background -(Android stops the process; a foreground service would keep it), and -read receipts per thread. +sharing on iOS (it needs a broadcast upload extension), and unread counts +per thread from the server (`unread_thread_notifications`); the thread +dots are worked out on the phone from receipts. ## Sync @@ -115,8 +119,20 @@ Element Call, with the same events, so the three can share a call. See messages (`io.element.call.encryption_keys`); keys are derived with HKDF and never ratcheted, like the web client's livekit-client. - A call of two shows the other person large and you in the corner; more - is a grid. Mute, camera, flip camera, leave. A "Join" banner in a room - with a call you're not in; the room list marks rooms with one. + is a grid. Mute, camera, flip camera, share the screen, leave. A "Join" + banner in a room with a call you're not in; the room list marks rooms + with one. +- Calls keep going with the app in the background: a foreground service + (`android/.../CallService.kt`, driven from `lib/matrix/call_service.dart`) + declares the microphone, the camera while it's on and the screen while + it's shared, which Android requires before it lets a backgrounded app + use them. Its notification shows the call's length; tap it to return to + the call, or hang up from it. Swiping the app away ends the call. +- Screen sharing (Android): Android asks for consent, the service adds + media projection, then LiveKit publishes the screen as a screen-share + track, which the web client and Element Call show as one. A shared + screen replaces its sharer's camera on their tile, shown whole; stopping + it from Android's own controls stops the share. ## Push notifications diff --git a/apps/mobile/android/app/build.gradle.kts b/apps/mobile/android/app/build.gradle.kts index 1ecfd9b..bc3311b 100644 --- a/apps/mobile/android/app/build.gradle.kts +++ b/apps/mobile/android/app/build.gradle.kts @@ -64,6 +64,8 @@ kotlin { dependencies { coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") + // The ongoing call notification (CallService). + implementation("androidx.core:core-ktx:1.17.0") } // UnifiedPush's connector depends on Tink's JVM artifact and diff --git a/apps/mobile/android/app/src/main/AndroidManifest.xml b/apps/mobile/android/app/src/main/AndroidManifest.xml index bac774e..2ec1da7 100644 --- a/apps/mobile/android/app/src/main/AndroidManifest.xml +++ b/apps/mobile/android/app/src/main/AndroidManifest.xml @@ -10,6 +10,12 @@ + + + + + @@ -47,6 +53,10 @@ + Unit)? = null + + private var running: CallService? = null + + /** + * Starts the service, or updates the running one. False when there + * is nothing to keep (no microphone, camera or screen permission), + * or Android refused. + */ + fun update( + context: Context, + title: String, + startedAt: Long, + camera: Boolean, + screen: Boolean, + ): Boolean { + val service = running + if (service != null) { + return service.show(title, startedAt, camera, screen) + } + if (types(context, camera, screen) == 0) return false + val intent = Intent(context, CallService::class.java) + .setAction(ACTION_UPDATE) + .putExtra(EXTRA_TITLE, title) + .putExtra(EXTRA_STARTED_AT, startedAt) + .putExtra(EXTRA_CAMERA, camera) + .putExtra(EXTRA_SCREEN, screen) + return try { + ContextCompat.startForegroundService(context, intent) + true + } catch (e: Exception) { + // Android 12+ refuses one started from the background. + Log.w(TAG, "Could not start the call service", e) + false + } + } + + fun stop(context: Context) { + context.stopService(Intent(context, CallService::class.java)) + } + + /** The foreground service types for what the call uses. */ + private fun types(context: Context, camera: Boolean, screen: Boolean): Int { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + // Before Android 11 only media projection is checked. + return if (screen || Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION + } else { + ServiceInfo.FOREGROUND_SERVICE_TYPE_MANIFEST + } + } + fun granted(permission: String) = ContextCompat.checkSelfPermission( + context, + permission, + ) == PackageManager.PERMISSION_GRANTED + var types = 0 + if (granted(Manifest.permission.RECORD_AUDIO)) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } + if (camera && granted(Manifest.permission.CAMERA)) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA + } + if (screen) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION + } + return types + } + } + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + if (intent?.action == ACTION_HANG_UP) { + onHangUp?.invoke() + if (running == null) stopSelf() + return START_NOT_STICKY + } + running = this + val shown = show( + intent?.getStringExtra(EXTRA_TITLE) ?: "", + intent?.getLongExtra(EXTRA_STARTED_AT, System.currentTimeMillis()) + ?: System.currentTimeMillis(), + intent?.getBooleanExtra(EXTRA_CAMERA, false) ?: false, + intent?.getBooleanExtra(EXTRA_SCREEN, false) ?: false, + ) + if (!shown) stopSelf() + return START_NOT_STICKY + } + + /** The app was swiped away: the call goes with it. */ + override fun onTaskRemoved(rootIntent: Intent?) { + onHangUp?.invoke() + stopSelf() + } + + override fun onDestroy() { + if (running == this) running = null + super.onDestroy() + } + + private fun show(title: String, startedAt: Long, camera: Boolean, screen: Boolean): Boolean { + val types = types(this, camera, screen) + if (types == 0) { + stopSelf() + return false + } + createChannel() + return try { + ServiceCompat.startForeground( + this, + NOTIFICATION_ID, + notification(title, startedAt, screen), + types, + ) + true + } catch (e: Exception) { + // Android 14 refuses camera and microphone types from the + // background, and media projection without consent. + Log.w(TAG, "Could not show the call notification", e) + false + } + } + + private fun notification(title: String, startedAt: Long, screen: Boolean) = + NotificationCompat.Builder(this, CHANNEL) + .setSmallIcon(R.drawable.ic_notification) + .setColor(0xFF00A389.toInt()) + .setContentTitle(title) + .setContentText(if (screen) "Sharing your screen" else "Call in progress") + .setContentIntent( + PendingIntent.getActivity( + this, + 0, + Intent(this, MainActivity::class.java) + .setAction(ACTION_OPEN) + .addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ), + ) + .setStyle( + NotificationCompat.CallStyle.forOngoingCall( + Person.Builder().setName(title).setImportant(true).build(), + PendingIntent.getService( + this, + 1, + Intent(this, CallService::class.java).setAction(ACTION_HANG_UP), + PendingIntent.FLAG_IMMUTABLE, + ), + ), + ) + .setCategory(NotificationCompat.CATEGORY_CALL) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setSilent(true) + .setWhen(startedAt) + .setUsesChronometer(true) + .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) + .build() + + private fun createChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + val manager = getSystemService(NotificationManager::class.java) + if (manager.getNotificationChannel(CHANNEL) != null) return + manager.createNotificationChannel( + NotificationChannel(CHANNEL, "Ongoing call", NotificationManager.IMPORTANCE_LOW).apply { + description = "The call you are in, while the app is in the background" + setShowBadge(false) + }, + ) + } +} diff --git a/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt b/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt index e3792c6..5798e54 100644 --- a/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt +++ b/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt @@ -1,5 +1,49 @@ package io.github.dariuszjaniak.warp +import android.content.Intent import io.flutter.embedding.android.FlutterActivity +import io.flutter.embedding.engine.FlutterEngine +import io.flutter.plugin.common.MethodChannel -class MainActivity : FlutterActivity() +class MainActivity : FlutterActivity() { + private var calls: MethodChannel? = null + + /** The call service's side of `warp/call_service`; see lib/matrix/call_service.dart. */ + override fun configureFlutterEngine(flutterEngine: FlutterEngine) { + super.configureFlutterEngine(flutterEngine) + val channel = MethodChannel(flutterEngine.dartExecutor.binaryMessenger, "warp/call_service") + channel.setMethodCallHandler { call, result -> + when (call.method) { + "update" -> result.success( + CallService.update( + this, + call.argument("title") ?: "", + call.argument("startedAt")?.toLong() ?: System.currentTimeMillis(), + call.argument("camera") ?: false, + call.argument("screen") ?: false, + ), + ) + "stop" -> { + CallService.stop(this) + result.success(null) + } + else -> result.notImplemented() + } + } + CallService.onHangUp = { runOnUiThread { channel.invokeMethod("hangUp", null) } } + calls = channel + } + + override fun onNewIntent(intent: Intent) { + super.onNewIntent(intent) + if (intent.action == CallService.ACTION_OPEN) calls?.invokeMethod("openCall", null) + } + + override fun onDestroy() { + // The Dart side, and so the call, goes with the activity: so does + // the notification. (Configuration changes don't destroy it.) + CallService.onHangUp = null + CallService.stop(this) + super.onDestroy() + } +} diff --git a/apps/mobile/lib/app.dart b/apps/mobile/lib/app.dart index 50ade1e..edc4d14 100644 --- a/apps/mobile/lib/app.dart +++ b/apps/mobile/lib/app.dart @@ -76,6 +76,7 @@ class _SignedInState extends ConsumerState<_SignedIn> ..add(client.onKeyVerificationRequest.stream.listen(_onVerification)); final calls = ref.read(callsProvider); calls.addListener(_onCalls); + _subs.add(calls.openRequests.listen((_) => _openCall())); _subs.add( calls.errors.stream.listen( (message) => messengerKey.currentState?.showSnackBar( @@ -134,6 +135,18 @@ class _SignedInState extends ConsumerState<_SignedIn> nav.push(RoomScreen.route(roomId)); } + /// Shows the call screen, unless it is already on top. + void _openCall() { + final nav = navigatorKey.currentState; + if (nav == null || !ref.read(callsProvider).inCall) return; + var onTop = false; + nav.popUntil((route) { + onTop = route.settings.name == 'call'; + return true; + }); + if (!onTop) nav.push(CallScreen.route()); + } + void _onUia(UiaRequest request) { final context = navigatorKey.currentContext; if (context == null) return request.cancel(); diff --git a/apps/mobile/lib/matrix/call_service.dart b/apps/mobile/lib/matrix/call_service.dart new file mode 100644 index 0000000..6a572e7 --- /dev/null +++ b/apps/mobile/lib/matrix/call_service.dart @@ -0,0 +1,61 @@ +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:matrix/matrix.dart'; + +/// The Android foreground service that keeps a call going with the app in +/// the background (android/.../CallService.kt): without it Android cuts a +/// backgrounded app off from the microphone and camera, and screen sharing +/// can't start at all. Its notification shows the call, opens it and hangs +/// up. Nothing on other platforms. +class CallService { + static const _channel = MethodChannel('warp/call_service'); + + /// Hang up from the notification, and tapping it. + static void listen({ + required void Function() onHangUp, + required void Function() onOpen, + }) { + if (!Platform.isAndroid) return; + _channel.setMethodCallHandler((call) async { + switch (call.method) { + case 'hangUp': + onHangUp(); + case 'openCall': + onOpen(); + } + }); + } + + /// Starts the service, or changes what it declares: the camera while it + /// is on, the screen while it is shared. False if Android refused. + static Future update({ + required String title, + required DateTime startedAt, + required bool camera, + required bool screen, + }) async { + if (!Platform.isAndroid) return true; + try { + return await _channel.invokeMethod('update', { + 'title': title, + 'startedAt': startedAt.millisecondsSinceEpoch, + 'camera': camera, + 'screen': screen, + }) ?? + false; + } catch (e) { + Logs().w('Could not start the call service', e); + return false; + } + } + + static Future stop() async { + if (!Platform.isAndroid) return; + try { + await _channel.invokeMethod('stop'); + } catch (e) { + Logs().w('Could not stop the call service', e); + } + } +} diff --git a/apps/mobile/lib/matrix/calls.dart b/apps/mobile/lib/matrix/calls.dart index c911cf3..366f382 100644 --- a/apps/mobile/lib/matrix/calls.dart +++ b/apps/mobile/lib/matrix/calls.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'dart:math'; import 'package:flutter/foundation.dart'; @@ -7,9 +8,11 @@ import 'package:http/http.dart' as http; import 'package:livekit_client/livekit_client.dart' as lk; import 'package:matrix/matrix.dart'; +import 'call_service.dart'; import 'notifications.dart'; import 'package:webrtc_interface/webrtc_interface.dart' as rtc; -import 'package:flutter_webrtc/flutter_webrtc.dart' show frameCryptorFactory; +import 'package:flutter_webrtc/flutter_webrtc.dart' + show Helper, frameCryptorFactory; /// Voice and video calls: MatrixRTC (MSC4143) with a LiveKit SFU, the way /// the web client and Element Call do them, so the three can share a call. @@ -31,6 +34,10 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' show frameCryptorFactory; /// (`org.matrix.msc4075.rtc.notification`); declining answers it /// (`org.matrix.msc4310.rtc.decline`). /// +/// While this device is in a call, a foreground service on Android +/// ([CallService]) keeps it going with the app in the background, and lets +/// it share the screen. +/// /// One call at a time. class Calls extends ChangeNotifier { Calls(this.client) { @@ -38,6 +45,7 @@ class Calls extends ChangeNotifier { ..add(client.onSync.stream.listen((_) => _refresh())) ..add(client.onTimelineEvent.stream.listen(_onTimelineEvent)) ..add(client.onToDeviceEvent.stream.listen(_onToDevice)); + CallService.listen(onHangUp: leave, onOpen: () => _openRequests.add(null)); _expiryTimer = Timer.periodic( const Duration(seconds: 30), (_) => _refresh(), @@ -90,6 +98,11 @@ class Calls extends ChangeNotifier { /// Errors to show once, then cleared. final errors = StreamController.broadcast(); + final _openRequests = StreamController.broadcast(); + + /// The ongoing call notification was tapped: show the call. + Stream get openRequests => _openRequests.stream; + bool get inCall => roomId != null; int inCallCount(String roomId) => active[roomId] ?? 0; @@ -419,6 +432,9 @@ class Calls extends ChangeNotifier { errors.add("Can't use the camera: $e"); } } + // Now that the microphone and camera are asked for, the service can + // declare them. + _mediaStarted = true; notifyListeners(); } catch (e, s) { Logs().e('Joining the call failed', e, s); @@ -438,6 +454,8 @@ class Calls extends ChangeNotifier { phase = null; joinedAt = null; ringingOut = false; + _mediaStarted = false; + _sharingScreen = false; _ringOutTimer?.cancel(); _delayTimer?.cancel(); _keyProvider = null; @@ -728,6 +746,11 @@ class Calls extends ChangeNotifier { bool get micOn => livekit?.localParticipant?.isMicrophoneEnabled() ?? false; bool get cameraOn => livekit?.localParticipant?.isCameraEnabled() ?? false; + bool get screenSharing => + livekit?.localParticipant?.isScreenShareEnabled() ?? false; + + /// Screen sharing: Android only (iOS needs a broadcast extension). + bool get canShareScreen => Platform.isAndroid; Future toggleMic() async { final p = livekit?.localParticipant; @@ -743,6 +766,53 @@ class Calls extends ChangeNotifier { notifyListeners(); } + /// Starts or stops sharing the screen. Android asks the user first, and + /// the call service must declare media projection before the capture + /// starts. + Future toggleScreenShare() async { + final p = livekit?.localParticipant; + if (p == null) return; + if (p.isScreenShareEnabled()) { + await _stopScreenShare(); + return; + } + if (!canShareScreen) return; + try { + if (!await Helper.requestCapturePermission()) return; + _sharingScreen = true; + if (!await _syncService()) { + _sharingScreen = false; + errors.add("Android didn't let the app share the screen."); + return; + } + final pub = await p.setScreenShareEnabled(true); + // Stopped from Android's own controls: unpublish it too. LiveKit + // listens to the track's end as well, so its handler is kept. + final track = pub?.track?.mediaStreamTrack; + if (track != null) { + final previous = track.onEnded; + track.onEnded = () { + previous?.call(); + if (_sharingScreen) _stopScreenShare(); + }; + } + } catch (e) { + _sharingScreen = false; + errors.add("Can't share the screen: $e"); + } + notifyListeners(); + } + + Future _stopScreenShare() async { + _sharingScreen = false; + try { + await livekit?.localParticipant?.setScreenShareEnabled(false); + } catch (e) { + Logs().w('Could not stop sharing the screen', e); + } + notifyListeners(); + } + /// Switches between the front and the back camera. Future flipCamera() async { final track = livekit?.localParticipant?.videoTrackPublications @@ -756,6 +826,50 @@ class Calls extends ChangeNotifier { notifyListeners(); } + // --- The call service ------------------------------------------------------ + + /// The microphone and camera have been asked for. + bool _mediaStarted = false; + + /// Screen sharing is on, or about to be. + bool _sharingScreen = false; + + /// What the service was last told, so it is told only of changes. + String? _serviceState; + + @override + void notifyListeners() { + super.notifyListeners(); + _syncService(); + } + + /// Starts, updates or stops the call service to match the call. + Future _syncService() async { + final rid = roomId; + final since = joinedAt; + if (rid == null || since == null || !_mediaStarted) { + if (_serviceState != null) { + _serviceState = null; + await CallService.stop(); + } + return false; + } + final title = client.getRoomById(rid)?.getLocalizedDisplayname() ?? 'Call'; + final camera = cameraOn; + final screen = _sharingScreen || screenSharing; + final state = '$rid|$title|$camera|$screen'; + if (state == _serviceState) return true; + _serviceState = state; + final ok = await CallService.update( + title: title, + startedAt: since, + camera: camera, + screen: screen, + ); + if (!ok) _serviceState = null; + return ok; + } + @override void dispose() { for (final s in _subs) { @@ -763,7 +877,9 @@ class Calls extends ChangeNotifier { } _expiryTimer?.cancel(); leave(); + CallService.stop(); errors.close(); + _openRequests.close(); super.dispose(); } } diff --git a/apps/mobile/lib/matrix/thread_receipts.dart b/apps/mobile/lib/matrix/thread_receipts.dart new file mode 100644 index 0000000..b63a684 --- /dev/null +++ b/apps/mobile/lib/matrix/thread_receipts.dart @@ -0,0 +1,103 @@ +import 'package:matrix/matrix.dart'; + +/// Read receipts per thread (MSC3771, in the spec since v1.4): a receipt +/// with a `thread_id` says how far someone has read in that thread alone, +/// so reading a thread doesn't mark the room read, nor the room a thread. +/// The SDK keeps them apart already (`receiptState.byThread`) but only +/// sends unthreaded ones; the thread screen sends these. + +/// Our receipt in the thread under [rootId], public or private, whichever +/// is further. +LatestReceiptStateData? ownThreadReceipt(Room room, String rootId) => + room.receiptState.byThread[rootId]?.latestOwnReceipt; + +/// Marks the thread under [rootId] read up to [eventId], a reply in it. +/// The private receipt always; the public one too unless the user keeps +/// receipts private, as for the room. +Future markThreadRead(Room room, String rootId, String eventId) async { + if (ownThreadReceipt(room, rootId)?.eventId == eventId) return; + final client = room.client; + await client.postReceipt( + room.id, + ReceiptType.mReadPrivate, + eventId, + threadId: rootId, + ); + if (client.receiptsPublicByDefault) { + await client.postReceipt( + room.id, + ReceiptType.mRead, + eventId, + threadId: rootId, + ); + } +} + +/// The other people whose receipt in the thread under [rootId] is on +/// [eventId], with those whose unthreaded receipt is. +Set threadReaders(Room room, String rootId, String eventId) { + final me = room.client.userID; + final state = room.receiptState; + return { + for (final e + in state.byThread[rootId]?.otherUsers.entries ?? + const >[]) + if (e.value.eventId == eventId && e.key != me) e.key, + for (final e in state.global.otherUsers.entries) + if (e.value.eventId == eventId && e.key != me) e.key, + }; +} + +/// The newest reply in a thread, as far as this device knows: the +/// server's summary on the root, or a newer one that came in since. +({String eventId, String senderId, DateTime time})? latestThreadReply( + Event root, [ + Timeline? timeline, +]) { + ({String eventId, String senderId, DateTime time})? latest; + final bundled = root.unsigned + ?.tryGetMap('m.relations') + ?.tryGetMap(RelationshipTypes.thread) + ?.tryGetMap('latest_event'); + final id = bundled?.tryGet('event_id'); + final sender = bundled?.tryGet('sender'); + final ts = bundled?.tryGet('origin_server_ts'); + if (id != null && sender != null && ts != null) { + latest = ( + eventId: id, + senderId: sender, + time: DateTime.fromMillisecondsSinceEpoch(ts), + ); + } + if (timeline != null) { + for (final e in root.aggregatedEvents(timeline, RelationshipTypes.thread)) { + if (!e.status.isSynced) continue; + if (latest == null || e.originServerTs.isAfter(latest.time)) { + latest = ( + eventId: e.eventId, + senderId: e.senderId, + time: e.originServerTs, + ); + } + } + } + return latest; +} + +/// Whether the thread under [root] has a reply we haven't read: one from +/// someone else, newer than our receipt in the thread and than our +/// unthreaded one (which covers everything before it). +bool threadUnread(Event root, [Timeline? timeline]) { + final latest = latestThreadReply(root, timeline); + if (latest == null) return false; + final room = root.room; + if (latest.senderId == room.client.userID) return false; + final inThread = ownThreadReceipt(room, root.eventId); + if (inThread?.eventId == latest.eventId) return false; + final unthreaded = room.receiptState.global.latestOwnReceipt; + if (unthreaded?.eventId == latest.eventId) return false; + for (final r in [inThread, unthreaded]) { + if (r != null && !latest.time.isAfter(r.timestamp)) return false; + } + return true; +} diff --git a/apps/mobile/lib/screens/call_screen.dart b/apps/mobile/lib/screens/call_screen.dart index 6383c5f..b41a03a 100644 --- a/apps/mobile/lib/screens/call_screen.dart +++ b/apps/mobile/lib/screens/call_screen.dart @@ -11,8 +11,9 @@ import '../theme.dart'; import '../widgets/avatar.dart'; /// The call this device is in: the other person large and you in the -/// corner for a call of two, a grid of tiles for more; mute, camera, flip -/// and leave. "Calling…" while it rings the other person. +/// corner for a call of two, a grid of tiles for more (a shared screen in +/// place of its sharer's camera); mute, camera, flip, share the screen and +/// leave. "Calling…" while it rings the other person. class CallScreen extends ConsumerStatefulWidget { const CallScreen({super.key}); @@ -103,6 +104,7 @@ class _CallScreenState extends ConsumerState { avatar: user?.avatarUrl, local: local, video: video(screen) ?? video(camera), + screen: video(screen) != null, micMuted: mic == null || mic.muted, speaking: speakers.contains(p.identity), ); @@ -130,6 +132,7 @@ class _Tile { required this.avatar, required this.local, required this.video, + required this.screen, required this.micMuted, required this.speaking, }); @@ -140,6 +143,9 @@ class _Tile { final Uri? avatar; final bool local; final lk.VideoTrack? video; + + /// [video] is a shared screen: shown whole, never mirrored. + final bool screen; final bool micMuted; final bool speaking; } @@ -288,10 +294,10 @@ class _TileView extends StatelessWidget { child: video != null ? lk.VideoTrackRenderer( video, - fit: large - ? lk.VideoViewFit.cover + fit: tile.screen + ? lk.VideoViewFit.contain : lk.VideoViewFit.cover, - mirrorMode: tile.local + mirrorMode: tile.local && !tile.screen ? lk.VideoViewMirrorMode.mirror : lk.VideoViewMirrorMode.off, ) @@ -349,37 +355,40 @@ class _Controls extends StatelessWidget { VoidCallback? onTap, { bool active = true, Color? colour, - }) => Column( - mainAxisSize: MainAxisSize.min, - children: [ - Material( - color: colour ?? (active ? Colors.white24 : Colors.white), - shape: const CircleBorder(), - child: InkWell( - customBorder: const CircleBorder(), - onTap: onTap, - child: SizedBox.square( - dimension: 60, - child: Icon( - icon, - color: colour != null || active ? Colors.white : Colors.black, + }) => Expanded( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Material( + color: colour ?? (active ? Colors.white24 : Colors.white), + shape: const CircleBorder(), + child: InkWell( + customBorder: const CircleBorder(), + onTap: onTap, + child: SizedBox.square( + dimension: 56, + child: Icon( + icon, + color: colour != null || active ? Colors.white : Colors.black, + ), ), ), ), - ), - const SizedBox(height: 6), - Text( - label, - style: const TextStyle(color: Colors.white70, fontSize: 12), - ), - ], + const SizedBox(height: 6), + Text( + label, + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: const TextStyle(color: Colors.white70, fontSize: 12), + ), + ], + ), ); final connected = calls.phase == CallPhase.connected; return Padding( - padding: const EdgeInsets.fromLTRB(16, 12, 16, 20), + padding: const EdgeInsets.fromLTRB(8, 12, 8, 20), child: Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, children: [ round( calls.micOn ? Icons.mic : Icons.mic_off, @@ -399,6 +408,15 @@ class _Controls extends StatelessWidget { 'Flip', connected ? calls.flipCamera : null, ), + if (calls.canShareScreen) + round( + calls.screenSharing + ? Icons.stop_screen_share + : Icons.screen_share, + 'Screen', + connected ? calls.toggleScreenShare : null, + active: !calls.screenSharing, + ), round( Icons.call_end, 'Leave', diff --git a/apps/mobile/lib/screens/thread_list_screen.dart b/apps/mobile/lib/screens/thread_list_screen.dart index fd8642f..60bf648 100644 --- a/apps/mobile/lib/screens/thread_list_screen.dart +++ b/apps/mobile/lib/screens/thread_list_screen.dart @@ -3,13 +3,15 @@ import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:matrix/matrix.dart'; import '../matrix/client.dart'; +import '../matrix/thread_receipts.dart'; import '../state.dart'; import '../util/format.dart'; import '../widgets/avatar.dart'; import '../widgets/message_extras.dart'; import 'thread_screen.dart'; -/// A room's threads, the most recently active first (`/threads`). +/// A room's threads, the most recently active first (`/threads`); those +/// with replies you haven't read are bold, with a dot. class ThreadListScreen extends ConsumerStatefulWidget { const ThreadListScreen({super.key, required this.roomId}); final String roomId; @@ -41,6 +43,8 @@ class _ThreadListScreenState extends ConsumerState { @override Widget build(BuildContext context) { + // Receipts come with syncs: the unread marks follow them. + ref.watch(syncProvider); return Scaffold( appBar: AppBar(title: const Text('Threads')), body: RefreshIndicator( @@ -97,6 +101,7 @@ class _ThreadListScreenState extends ConsumerState { final when = latestTs == null ? root.originServerTs : DateTime.fromMillisecondsSinceEpoch(latestTs); + final unread = threadUnread(root); return ListTile( leading: Avatar( name: sender.calcDisplayname(), @@ -107,12 +112,23 @@ class _ThreadListScreenState extends ConsumerState { quoteText(root), maxLines: 2, overflow: TextOverflow.ellipsis, + style: unread + ? const TextStyle(fontWeight: FontWeight.w700) + : null, ), subtitle: Text( '${sender.calcDisplayname()} · ' '${count == 1 ? '1 reply' : '$count replies'} · ' '${formatListTime(when)}', ), + trailing: unread + ? Icon( + Icons.circle, + size: 10, + color: Theme.of(context).colorScheme.primary, + semanticLabel: 'Unread', + ) + : null, onTap: () => Navigator.of( context, ).push(ThreadScreen.route(widget.roomId, root.eventId)), diff --git a/apps/mobile/lib/screens/thread_screen.dart b/apps/mobile/lib/screens/thread_screen.dart index 07f6189..0ad720c 100644 --- a/apps/mobile/lib/screens/thread_screen.dart +++ b/apps/mobile/lib/screens/thread_screen.dart @@ -3,6 +3,7 @@ import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:matrix/matrix.dart'; import '../matrix/client.dart'; +import '../matrix/thread_receipts.dart'; import '../matrix/timeline.dart'; import '../state.dart'; import '../widgets/composer.dart'; @@ -15,6 +16,10 @@ import '../widgets/message.dart'; /// (which receives new replies, their sending status, edits and reactions /// like any event) and adds the older replies it doesn't have from /// `/relations`. +/// +/// While it is on screen and the app in front, it marks the thread read +/// with a receipt for the thread alone; "Read" under your replies comes +/// from the others' receipts in the thread. class ThreadScreen extends ConsumerStatefulWidget { const ThreadScreen({super.key, required this.roomId, required this.rootId}); final String roomId; @@ -29,7 +34,8 @@ class ThreadScreen extends ConsumerStatefulWidget { ConsumerState createState() => _ThreadScreenState(); } -class _ThreadScreenState extends ConsumerState { +class _ThreadScreenState extends ConsumerState + with WidgetsBindingObserver { Timeline? _timeline; Event? _root; Object? _error; @@ -42,17 +48,22 @@ class _ThreadScreenState extends ConsumerState { Event? _replyTo; final _scroll = ScrollController(); + /// The reply we last sent a receipt for, until the server echoes it. + String? _receiptSent; + Room get _room => ref.read(clientProvider).getRoomById(widget.roomId)!; @override void initState() { super.initState(); + WidgetsBinding.instance.addObserver(this); _scroll.addListener(_onScroll); _load(); } @override void dispose() { + WidgetsBinding.instance.removeObserver(this); _timeline?.cancelSubscriptions(); _scroll.dispose(); super.dispose(); @@ -64,6 +75,7 @@ class _ThreadScreenState extends ConsumerState { final timeline = await room.getTimeline( onUpdate: () { if (mounted) setState(() {}); + _markRead(); }, ); final root = @@ -78,11 +90,38 @@ class _ThreadScreenState extends ConsumerState { _root = root; }); await _loadOlder(); + _markRead(); } catch (e) { if (mounted) setState(() => _error = e); } } + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (state == AppLifecycleState.resumed) _markRead(); + } + + /// Sends a receipt in the thread for its newest reply, while the thread + /// is on screen and the app is in front. + void _markRead() { + if (!mounted || _timeline == null) return; + if (WidgetsBinding.instance.lifecycleState != AppLifecycleState.resumed) { + return; + } + final room = _room; + if (room.membership != Membership.join) return; + final newest = _replies().where((e) => e.status.isSynced).firstOrNull; + if (newest == null || newest.eventId == _receiptSent) return; + if (ownThreadReceipt(room, widget.rootId)?.eventId == newest.eventId) { + return; + } + _receiptSent = newest.eventId; + markThreadRead(room, widget.rootId, newest.eventId).catchError((Object e) { + Logs().w('Could not send a thread read receipt', e); + _receiptSent = null; + }); + } + Future _decrypt(Event event) async { final encryption = event.room.client.encryption; if (event.type != EventTypes.Encrypted || encryption == null) return event; @@ -171,6 +210,7 @@ class _ThreadScreenState extends ConsumerState { } else { final replies = _replies(); final entries = [...replies, ?root]; + final readUpTo = _readByOthersIndex(entries, room, widget.rootId); final items = projectTimeline([ for (final e in entries) ( @@ -217,7 +257,7 @@ class _ThreadScreenState extends ConsumerState { timeline: timeline, firstInGroup: firstInGroup, lastInGroup: lastInGroup, - readByOthers: false, + readByOthers: readUpTo != null && index >= readUpTo, onReply: (e) => setState(() => _replyTo = e), ), if (entries[index].eventId == widget.rootId) @@ -266,6 +306,21 @@ class _ThreadScreenState extends ConsumerState { } } +/// The index (newest first) of the newest entry someone else has read, +/// by their receipt in the thread or an unthreaded one, so your replies +/// from there back say "Read". +int? _readByOthersIndex(List entries, Room room, String rootId) { + for (var i = 0; i < entries.length; i++) { + final e = entries[i]; + if (!e.status.isSynced) continue; + if (threadReaders(room, rootId, e.eventId).isNotEmpty || + e.receipts.any((r) => r.user.id != room.client.userID)) { + return i; + } + } + return null; +} + class _ThreadDivider extends StatelessWidget { const _ThreadDivider(); diff --git a/apps/mobile/lib/widgets/message_extras.dart b/apps/mobile/lib/widgets/message_extras.dart index a1ca83f..0038ce5 100644 --- a/apps/mobile/lib/widgets/message_extras.dart +++ b/apps/mobile/lib/widgets/message_extras.dart @@ -2,6 +2,7 @@ import 'package:flutter/material.dart'; import 'package:matrix/matrix.dart'; import '../matrix/client.dart'; +import '../matrix/thread_receipts.dart'; import '../theme.dart'; import 'avatar.dart'; @@ -304,7 +305,8 @@ class QuoteBlock extends StatelessWidget { } } -/// Under a message that starts a thread: how many replies, opening it. +/// Under a message that starts a thread: how many replies, opening it; +/// a dot when it has replies you haven't read. class ThreadSummary extends StatelessWidget { const ThreadSummary({ super.key, @@ -340,6 +342,15 @@ class ThreadSummary extends StatelessWidget { fontWeight: FontWeight.w600, ), ), + if (threadUnread(event, timeline)) ...[ + const SizedBox(width: 6), + Icon( + Icons.circle, + size: 8, + color: c.tealText, + semanticLabel: 'Unread', + ), + ], ], ), ),