From 12550c407855363dcc16042f72818cc681aa274a Mon Sep 17 00:00:00 2001 From: tajniak81 <13187254+tajniak81@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:22:56 +0200 Subject: [PATCH] Mobile: calls in the background, screen sharing, read receipts per thread - Calls keep going with the app in the background: a foreground service (CallService.kt, driven from lib/matrix/call_service.dart) declares the microphone, the camera while it's on and the screen while it's shared. Its notification (CallStyle) shows the call's length, opens the call and hangs up. Swiping the app away ends the call. - Screen sharing on Android: consent, then media projection on the service, then LiveKit publishes a screen-share track. A shared screen replaces the sharer's camera on their tile, shown whole and unmirrored; stopping it from Android's controls stops the share. - Read receipts per thread (MSC3771): an open thread sends receipts with thread_id, "Read" under replies comes from receipts in the thread, and threads with unread replies get a dot on "N replies" and in the list. Not tested on a device yet. README updated. Co-Authored-By: Claude Opus 5.5 --- apps/mobile/README.md | 26 ++- apps/mobile/android/app/build.gradle.kts | 2 + .../android/app/src/main/AndroidManifest.xml | 10 + .../github/dariuszjaniak/warp/CallService.kt | 218 ++++++++++++++++++ .../github/dariuszjaniak/warp/MainActivity.kt | 46 +++- apps/mobile/lib/app.dart | 13 ++ apps/mobile/lib/matrix/call_service.dart | 61 +++++ apps/mobile/lib/matrix/calls.dart | 118 +++++++++- apps/mobile/lib/matrix/thread_receipts.dart | 103 +++++++++ apps/mobile/lib/screens/call_screen.dart | 74 +++--- .../lib/screens/thread_list_screen.dart | 18 +- apps/mobile/lib/screens/thread_screen.dart | 59 ++++- apps/mobile/lib/widgets/message_extras.dart | 13 +- 13 files changed, 722 insertions(+), 39 deletions(-) create mode 100644 apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/CallService.kt create mode 100644 apps/mobile/lib/matrix/call_service.dart create mode 100644 apps/mobile/lib/matrix/thread_receipts.dart diff --git a/apps/mobile/README.md b/apps/mobile/README.md index 12f1793..fca88ba 100644 --- a/apps/mobile/README.md +++ b/apps/mobile/README.md @@ -39,6 +39,10 @@ radii), the Outfit and Figtree fonts, the Warp symbol. - Threads: *Reply in thread* opens the thread, its root and replies, with a composer that sends into it. Replies stay out of the main timeline; the root shows "N replies". Room info lists a room's threads. + Read receipts per thread (MSC3771): an open thread marks itself read + with a receipt for the thread alone, "Read" under your replies comes + from the others' receipts in the thread, and threads with replies you + haven't read get a dot (on "N replies" and in the thread list). - Spaces: the room list's title picks a space, which narrows the list to its rooms (and its subspaces'). A space's browser (`/hierarchy`) shows the rooms you're in and the ones you can join; admins add existing or @@ -67,9 +71,9 @@ radii), the Outfit and Figtree fonts, the Warp symbol. - Settings: display name, avatar, theme (system, light, dark). **Not yet:** a per-message trust shield like the web client's, screen -sharing in calls, calls that keep going with the app in the background -(Android stops the process; a foreground service would keep it), and -read receipts per thread. +sharing on iOS (it needs a broadcast upload extension), and unread counts +per thread from the server (`unread_thread_notifications`); the thread +dots are worked out on the phone from receipts. ## Sync @@ -115,8 +119,20 @@ Element Call, with the same events, so the three can share a call. See messages (`io.element.call.encryption_keys`); keys are derived with HKDF and never ratcheted, like the web client's livekit-client. - A call of two shows the other person large and you in the corner; more - is a grid. Mute, camera, flip camera, leave. A "Join" banner in a room - with a call you're not in; the room list marks rooms with one. + is a grid. Mute, camera, flip camera, share the screen, leave. A "Join" + banner in a room with a call you're not in; the room list marks rooms + with one. +- Calls keep going with the app in the background: a foreground service + (`android/.../CallService.kt`, driven from `lib/matrix/call_service.dart`) + declares the microphone, the camera while it's on and the screen while + it's shared, which Android requires before it lets a backgrounded app + use them. Its notification shows the call's length; tap it to return to + the call, or hang up from it. Swiping the app away ends the call. +- Screen sharing (Android): Android asks for consent, the service adds + media projection, then LiveKit publishes the screen as a screen-share + track, which the web client and Element Call show as one. A shared + screen replaces its sharer's camera on their tile, shown whole; stopping + it from Android's own controls stops the share. ## Push notifications diff --git a/apps/mobile/android/app/build.gradle.kts b/apps/mobile/android/app/build.gradle.kts index 1ecfd9b..bc3311b 100644 --- a/apps/mobile/android/app/build.gradle.kts +++ b/apps/mobile/android/app/build.gradle.kts @@ -64,6 +64,8 @@ kotlin { dependencies { coreLibraryDesugaring("com.android.tools:desugar_jdk_libs:2.1.5") + // The ongoing call notification (CallService). + implementation("androidx.core:core-ktx:1.17.0") } // UnifiedPush's connector depends on Tink's JVM artifact and diff --git a/apps/mobile/android/app/src/main/AndroidManifest.xml b/apps/mobile/android/app/src/main/AndroidManifest.xml index bac774e..2ec1da7 100644 --- a/apps/mobile/android/app/src/main/AndroidManifest.xml +++ b/apps/mobile/android/app/src/main/AndroidManifest.xml @@ -10,6 +10,12 @@ + + + + + @@ -47,6 +53,10 @@ + Unit)? = null + + private var running: CallService? = null + + /** + * Starts the service, or updates the running one. False when there + * is nothing to keep (no microphone, camera or screen permission), + * or Android refused. + */ + fun update( + context: Context, + title: String, + startedAt: Long, + camera: Boolean, + screen: Boolean, + ): Boolean { + val service = running + if (service != null) { + return service.show(title, startedAt, camera, screen) + } + if (types(context, camera, screen) == 0) return false + val intent = Intent(context, CallService::class.java) + .setAction(ACTION_UPDATE) + .putExtra(EXTRA_TITLE, title) + .putExtra(EXTRA_STARTED_AT, startedAt) + .putExtra(EXTRA_CAMERA, camera) + .putExtra(EXTRA_SCREEN, screen) + return try { + ContextCompat.startForegroundService(context, intent) + true + } catch (e: Exception) { + // Android 12+ refuses one started from the background. + Log.w(TAG, "Could not start the call service", e) + false + } + } + + fun stop(context: Context) { + context.stopService(Intent(context, CallService::class.java)) + } + + /** The foreground service types for what the call uses. */ + private fun types(context: Context, camera: Boolean, screen: Boolean): Int { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) { + // Before Android 11 only media projection is checked. + return if (screen || Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) { + ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION + } else { + ServiceInfo.FOREGROUND_SERVICE_TYPE_MANIFEST + } + } + fun granted(permission: String) = ContextCompat.checkSelfPermission( + context, + permission, + ) == PackageManager.PERMISSION_GRANTED + var types = 0 + if (granted(Manifest.permission.RECORD_AUDIO)) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + } + if (camera && granted(Manifest.permission.CAMERA)) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA + } + if (screen) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION + } + return types + } + } + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + if (intent?.action == ACTION_HANG_UP) { + onHangUp?.invoke() + if (running == null) stopSelf() + return START_NOT_STICKY + } + running = this + val shown = show( + intent?.getStringExtra(EXTRA_TITLE) ?: "", + intent?.getLongExtra(EXTRA_STARTED_AT, System.currentTimeMillis()) + ?: System.currentTimeMillis(), + intent?.getBooleanExtra(EXTRA_CAMERA, false) ?: false, + intent?.getBooleanExtra(EXTRA_SCREEN, false) ?: false, + ) + if (!shown) stopSelf() + return START_NOT_STICKY + } + + /** The app was swiped away: the call goes with it. */ + override fun onTaskRemoved(rootIntent: Intent?) { + onHangUp?.invoke() + stopSelf() + } + + override fun onDestroy() { + if (running == this) running = null + super.onDestroy() + } + + private fun show(title: String, startedAt: Long, camera: Boolean, screen: Boolean): Boolean { + val types = types(this, camera, screen) + if (types == 0) { + stopSelf() + return false + } + createChannel() + return try { + ServiceCompat.startForeground( + this, + NOTIFICATION_ID, + notification(title, startedAt, screen), + types, + ) + true + } catch (e: Exception) { + // Android 14 refuses camera and microphone types from the + // background, and media projection without consent. + Log.w(TAG, "Could not show the call notification", e) + false + } + } + + private fun notification(title: String, startedAt: Long, screen: Boolean) = + NotificationCompat.Builder(this, CHANNEL) + .setSmallIcon(R.drawable.ic_notification) + .setColor(0xFF00A389.toInt()) + .setContentTitle(title) + .setContentText(if (screen) "Sharing your screen" else "Call in progress") + .setContentIntent( + PendingIntent.getActivity( + this, + 0, + Intent(this, MainActivity::class.java) + .setAction(ACTION_OPEN) + .addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ), + ) + .setStyle( + NotificationCompat.CallStyle.forOngoingCall( + Person.Builder().setName(title).setImportant(true).build(), + PendingIntent.getService( + this, + 1, + Intent(this, CallService::class.java).setAction(ACTION_HANG_UP), + PendingIntent.FLAG_IMMUTABLE, + ), + ), + ) + .setCategory(NotificationCompat.CATEGORY_CALL) + .setOngoing(true) + .setOnlyAlertOnce(true) + .setSilent(true) + .setWhen(startedAt) + .setUsesChronometer(true) + .setForegroundServiceBehavior(NotificationCompat.FOREGROUND_SERVICE_IMMEDIATE) + .build() + + private fun createChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + val manager = getSystemService(NotificationManager::class.java) + if (manager.getNotificationChannel(CHANNEL) != null) return + manager.createNotificationChannel( + NotificationChannel(CHANNEL, "Ongoing call", NotificationManager.IMPORTANCE_LOW).apply { + description = "The call you are in, while the app is in the background" + setShowBadge(false) + }, + ) + } +} diff --git a/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt b/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt index e3792c6..5798e54 100644 --- a/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt +++ b/apps/mobile/android/app/src/main/kotlin/io/github/dariuszjaniak/warp/MainActivity.kt @@ -1,5 +1,49 @@ package io.github.dariuszjaniak.warp +import android.content.Intent import io.flutter.embedding.android.FlutterActivity +import io.flutter.embedding.engine.FlutterEngine +import io.flutter.plugin.common.MethodChannel -class MainActivity : FlutterActivity() +class MainActivity : FlutterActivity() { + private var calls: MethodChannel? = null + + /** The call service's side of `warp/call_service`; see lib/matrix/call_service.dart. */ + override fun configureFlutterEngine(flutterEngine: FlutterEngine) { + super.configureFlutterEngine(flutterEngine) + val channel = MethodChannel(flutterEngine.dartExecutor.binaryMessenger, "warp/call_service") + channel.setMethodCallHandler { call, result -> + when (call.method) { + "update" -> result.success( + CallService.update( + this, + call.argument("title") ?: "", + call.argument("startedAt")?.toLong() ?: System.currentTimeMillis(), + call.argument("camera") ?: false, + call.argument("screen") ?: false, + ), + ) + "stop" -> { + CallService.stop(this) + result.success(null) + } + else -> result.notImplemented() + } + } + CallService.onHangUp = { runOnUiThread { channel.invokeMethod("hangUp", null) } } + calls = channel + } + + override fun onNewIntent(intent: Intent) { + super.onNewIntent(intent) + if (intent.action == CallService.ACTION_OPEN) calls?.invokeMethod("openCall", null) + } + + override fun onDestroy() { + // The Dart side, and so the call, goes with the activity: so does + // the notification. (Configuration changes don't destroy it.) + CallService.onHangUp = null + CallService.stop(this) + super.onDestroy() + } +} diff --git a/apps/mobile/lib/app.dart b/apps/mobile/lib/app.dart index 50ade1e..edc4d14 100644 --- a/apps/mobile/lib/app.dart +++ b/apps/mobile/lib/app.dart @@ -76,6 +76,7 @@ class _SignedInState extends ConsumerState<_SignedIn> ..add(client.onKeyVerificationRequest.stream.listen(_onVerification)); final calls = ref.read(callsProvider); calls.addListener(_onCalls); + _subs.add(calls.openRequests.listen((_) => _openCall())); _subs.add( calls.errors.stream.listen( (message) => messengerKey.currentState?.showSnackBar( @@ -134,6 +135,18 @@ class _SignedInState extends ConsumerState<_SignedIn> nav.push(RoomScreen.route(roomId)); } + /// Shows the call screen, unless it is already on top. + void _openCall() { + final nav = navigatorKey.currentState; + if (nav == null || !ref.read(callsProvider).inCall) return; + var onTop = false; + nav.popUntil((route) { + onTop = route.settings.name == 'call'; + return true; + }); + if (!onTop) nav.push(CallScreen.route()); + } + void _onUia(UiaRequest request) { final context = navigatorKey.currentContext; if (context == null) return request.cancel(); diff --git a/apps/mobile/lib/matrix/call_service.dart b/apps/mobile/lib/matrix/call_service.dart new file mode 100644 index 0000000..6a572e7 --- /dev/null +++ b/apps/mobile/lib/matrix/call_service.dart @@ -0,0 +1,61 @@ +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:matrix/matrix.dart'; + +/// The Android foreground service that keeps a call going with the app in +/// the background (android/.../CallService.kt): without it Android cuts a +/// backgrounded app off from the microphone and camera, and screen sharing +/// can't start at all. Its notification shows the call, opens it and hangs +/// up. Nothing on other platforms. +class CallService { + static const _channel = MethodChannel('warp/call_service'); + + /// Hang up from the notification, and tapping it. + static void listen({ + required void Function() onHangUp, + required void Function() onOpen, + }) { + if (!Platform.isAndroid) return; + _channel.setMethodCallHandler((call) async { + switch (call.method) { + case 'hangUp': + onHangUp(); + case 'openCall': + onOpen(); + } + }); + } + + /// Starts the service, or changes what it declares: the camera while it + /// is on, the screen while it is shared. False if Android refused. + static Future update({ + required String title, + required DateTime startedAt, + required bool camera, + required bool screen, + }) async { + if (!Platform.isAndroid) return true; + try { + return await _channel.invokeMethod('update', { + 'title': title, + 'startedAt': startedAt.millisecondsSinceEpoch, + 'camera': camera, + 'screen': screen, + }) ?? + false; + } catch (e) { + Logs().w('Could not start the call service', e); + return false; + } + } + + static Future stop() async { + if (!Platform.isAndroid) return; + try { + await _channel.invokeMethod('stop'); + } catch (e) { + Logs().w('Could not stop the call service', e); + } + } +} diff --git a/apps/mobile/lib/matrix/calls.dart b/apps/mobile/lib/matrix/calls.dart index c911cf3..366f382 100644 --- a/apps/mobile/lib/matrix/calls.dart +++ b/apps/mobile/lib/matrix/calls.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'dart:math'; import 'package:flutter/foundation.dart'; @@ -7,9 +8,11 @@ import 'package:http/http.dart' as http; import 'package:livekit_client/livekit_client.dart' as lk; import 'package:matrix/matrix.dart'; +import 'call_service.dart'; import 'notifications.dart'; import 'package:webrtc_interface/webrtc_interface.dart' as rtc; -import 'package:flutter_webrtc/flutter_webrtc.dart' show frameCryptorFactory; +import 'package:flutter_webrtc/flutter_webrtc.dart' + show Helper, frameCryptorFactory; /// Voice and video calls: MatrixRTC (MSC4143) with a LiveKit SFU, the way /// the web client and Element Call do them, so the three can share a call. @@ -31,6 +34,10 @@ import 'package:flutter_webrtc/flutter_webrtc.dart' show frameCryptorFactory; /// (`org.matrix.msc4075.rtc.notification`); declining answers it /// (`org.matrix.msc4310.rtc.decline`). /// +/// While this device is in a call, a foreground service on Android +/// ([CallService]) keeps it going with the app in the background, and lets +/// it share the screen. +/// /// One call at a time. class Calls extends ChangeNotifier { Calls(this.client) { @@ -38,6 +45,7 @@ class Calls extends ChangeNotifier { ..add(client.onSync.stream.listen((_) => _refresh())) ..add(client.onTimelineEvent.stream.listen(_onTimelineEvent)) ..add(client.onToDeviceEvent.stream.listen(_onToDevice)); + CallService.listen(onHangUp: leave, onOpen: () => _openRequests.add(null)); _expiryTimer = Timer.periodic( const Duration(seconds: 30), (_) => _refresh(), @@ -90,6 +98,11 @@ class Calls extends ChangeNotifier { /// Errors to show once, then cleared. final errors = StreamController.broadcast(); + final _openRequests = StreamController.broadcast(); + + /// The ongoing call notification was tapped: show the call. + Stream get openRequests => _openRequests.stream; + bool get inCall => roomId != null; int inCallCount(String roomId) => active[roomId] ?? 0; @@ -419,6 +432,9 @@ class Calls extends ChangeNotifier { errors.add("Can't use the camera: $e"); } } + // Now that the microphone and camera are asked for, the service can + // declare them. + _mediaStarted = true; notifyListeners(); } catch (e, s) { Logs().e('Joining the call failed', e, s); @@ -438,6 +454,8 @@ class Calls extends ChangeNotifier { phase = null; joinedAt = null; ringingOut = false; + _mediaStarted = false; + _sharingScreen = false; _ringOutTimer?.cancel(); _delayTimer?.cancel(); _keyProvider = null; @@ -728,6 +746,11 @@ class Calls extends ChangeNotifier { bool get micOn => livekit?.localParticipant?.isMicrophoneEnabled() ?? false; bool get cameraOn => livekit?.localParticipant?.isCameraEnabled() ?? false; + bool get screenSharing => + livekit?.localParticipant?.isScreenShareEnabled() ?? false; + + /// Screen sharing: Android only (iOS needs a broadcast extension). + bool get canShareScreen => Platform.isAndroid; Future toggleMic() async { final p = livekit?.localParticipant; @@ -743,6 +766,53 @@ class Calls extends ChangeNotifier { notifyListeners(); } + /// Starts or stops sharing the screen. Android asks the user first, and + /// the call service must declare media projection before the capture + /// starts. + Future toggleScreenShare() async { + final p = livekit?.localParticipant; + if (p == null) return; + if (p.isScreenShareEnabled()) { + await _stopScreenShare(); + return; + } + if (!canShareScreen) return; + try { + if (!await Helper.requestCapturePermission()) return; + _sharingScreen = true; + if (!await _syncService()) { + _sharingScreen = false; + errors.add("Android didn't let the app share the screen."); + return; + } + final pub = await p.setScreenShareEnabled(true); + // Stopped from Android's own controls: unpublish it too. LiveKit + // listens to the track's end as well, so its handler is kept. + final track = pub?.track?.mediaStreamTrack; + if (track != null) { + final previous = track.onEnded; + track.onEnded = () { + previous?.call(); + if (_sharingScreen) _stopScreenShare(); + }; + } + } catch (e) { + _sharingScreen = false; + errors.add("Can't share the screen: $e"); + } + notifyListeners(); + } + + Future _stopScreenShare() async { + _sharingScreen = false; + try { + await livekit?.localParticipant?.setScreenShareEnabled(false); + } catch (e) { + Logs().w('Could not stop sharing the screen', e); + } + notifyListeners(); + } + /// Switches between the front and the back camera. Future flipCamera() async { final track = livekit?.localParticipant?.videoTrackPublications @@ -756,6 +826,50 @@ class Calls extends ChangeNotifier { notifyListeners(); } + // --- The call service ------------------------------------------------------ + + /// The microphone and camera have been asked for. + bool _mediaStarted = false; + + /// Screen sharing is on, or about to be. + bool _sharingScreen = false; + + /// What the service was last told, so it is told only of changes. + String? _serviceState; + + @override + void notifyListeners() { + super.notifyListeners(); + _syncService(); + } + + /// Starts, updates or stops the call service to match the call. + Future _syncService() async { + final rid = roomId; + final since = joinedAt; + if (rid == null || since == null || !_mediaStarted) { + if (_serviceState != null) { + _serviceState = null; + await CallService.stop(); + } + return false; + } + final title = client.getRoomById(rid)?.getLocalizedDisplayname() ?? 'Call'; + final camera = cameraOn; + final screen = _sharingScreen || screenSharing; + final state = '$rid|$title|$camera|$screen'; + if (state == _serviceState) return true; + _serviceState = state; + final ok = await CallService.update( + title: title, + startedAt: since, + camera: camera, + screen: screen, + ); + if (!ok) _serviceState = null; + return ok; + } + @override void dispose() { for (final s in _subs) { @@ -763,7 +877,9 @@ class Calls extends ChangeNotifier { } _expiryTimer?.cancel(); leave(); + CallService.stop(); errors.close(); + _openRequests.close(); super.dispose(); } } diff --git a/apps/mobile/lib/matrix/thread_receipts.dart b/apps/mobile/lib/matrix/thread_receipts.dart new file mode 100644 index 0000000..b63a684 --- /dev/null +++ b/apps/mobile/lib/matrix/thread_receipts.dart @@ -0,0 +1,103 @@ +import 'package:matrix/matrix.dart'; + +/// Read receipts per thread (MSC3771, in the spec since v1.4): a receipt +/// with a `thread_id` says how far someone has read in that thread alone, +/// so reading a thread doesn't mark the room read, nor the room a thread. +/// The SDK keeps them apart already (`receiptState.byThread`) but only +/// sends unthreaded ones; the thread screen sends these. + +/// Our receipt in the thread under [rootId], public or private, whichever +/// is further. +LatestReceiptStateData? ownThreadReceipt(Room room, String rootId) => + room.receiptState.byThread[rootId]?.latestOwnReceipt; + +/// Marks the thread under [rootId] read up to [eventId], a reply in it. +/// The private receipt always; the public one too unless the user keeps +/// receipts private, as for the room. +Future markThreadRead(Room room, String rootId, String eventId) async { + if (ownThreadReceipt(room, rootId)?.eventId == eventId) return; + final client = room.client; + await client.postReceipt( + room.id, + ReceiptType.mReadPrivate, + eventId, + threadId: rootId, + ); + if (client.receiptsPublicByDefault) { + await client.postReceipt( + room.id, + ReceiptType.mRead, + eventId, + threadId: rootId, + ); + } +} + +/// The other people whose receipt in the thread under [rootId] is on +/// [eventId], with those whose unthreaded receipt is. +Set threadReaders(Room room, String rootId, String eventId) { + final me = room.client.userID; + final state = room.receiptState; + return { + for (final e + in state.byThread[rootId]?.otherUsers.entries ?? + const >[]) + if (e.value.eventId == eventId && e.key != me) e.key, + for (final e in state.global.otherUsers.entries) + if (e.value.eventId == eventId && e.key != me) e.key, + }; +} + +/// The newest reply in a thread, as far as this device knows: the +/// server's summary on the root, or a newer one that came in since. +({String eventId, String senderId, DateTime time})? latestThreadReply( + Event root, [ + Timeline? timeline, +]) { + ({String eventId, String senderId, DateTime time})? latest; + final bundled = root.unsigned + ?.tryGetMap('m.relations') + ?.tryGetMap(RelationshipTypes.thread) + ?.tryGetMap('latest_event'); + final id = bundled?.tryGet('event_id'); + final sender = bundled?.tryGet('sender'); + final ts = bundled?.tryGet('origin_server_ts'); + if (id != null && sender != null && ts != null) { + latest = ( + eventId: id, + senderId: sender, + time: DateTime.fromMillisecondsSinceEpoch(ts), + ); + } + if (timeline != null) { + for (final e in root.aggregatedEvents(timeline, RelationshipTypes.thread)) { + if (!e.status.isSynced) continue; + if (latest == null || e.originServerTs.isAfter(latest.time)) { + latest = ( + eventId: e.eventId, + senderId: e.senderId, + time: e.originServerTs, + ); + } + } + } + return latest; +} + +/// Whether the thread under [root] has a reply we haven't read: one from +/// someone else, newer than our receipt in the thread and than our +/// unthreaded one (which covers everything before it). +bool threadUnread(Event root, [Timeline? timeline]) { + final latest = latestThreadReply(root, timeline); + if (latest == null) return false; + final room = root.room; + if (latest.senderId == room.client.userID) return false; + final inThread = ownThreadReceipt(room, root.eventId); + if (inThread?.eventId == latest.eventId) return false; + final unthreaded = room.receiptState.global.latestOwnReceipt; + if (unthreaded?.eventId == latest.eventId) return false; + for (final r in [inThread, unthreaded]) { + if (r != null && !latest.time.isAfter(r.timestamp)) return false; + } + return true; +} diff --git a/apps/mobile/lib/screens/call_screen.dart b/apps/mobile/lib/screens/call_screen.dart index 6383c5f..b41a03a 100644 --- a/apps/mobile/lib/screens/call_screen.dart +++ b/apps/mobile/lib/screens/call_screen.dart @@ -11,8 +11,9 @@ import '../theme.dart'; import '../widgets/avatar.dart'; /// The call this device is in: the other person large and you in the -/// corner for a call of two, a grid of tiles for more; mute, camera, flip -/// and leave. "Calling…" while it rings the other person. +/// corner for a call of two, a grid of tiles for more (a shared screen in +/// place of its sharer's camera); mute, camera, flip, share the screen and +/// leave. "Calling…" while it rings the other person. class CallScreen extends ConsumerStatefulWidget { const CallScreen({super.key}); @@ -103,6 +104,7 @@ class _CallScreenState extends ConsumerState { avatar: user?.avatarUrl, local: local, video: video(screen) ?? video(camera), + screen: video(screen) != null, micMuted: mic == null || mic.muted, speaking: speakers.contains(p.identity), ); @@ -130,6 +132,7 @@ class _Tile { required this.avatar, required this.local, required this.video, + required this.screen, required this.micMuted, required this.speaking, }); @@ -140,6 +143,9 @@ class _Tile { final Uri? avatar; final bool local; final lk.VideoTrack? video; + + /// [video] is a shared screen: shown whole, never mirrored. + final bool screen; final bool micMuted; final bool speaking; } @@ -288,10 +294,10 @@ class _TileView extends StatelessWidget { child: video != null ? lk.VideoTrackRenderer( video, - fit: large - ? lk.VideoViewFit.cover + fit: tile.screen + ? lk.VideoViewFit.contain : lk.VideoViewFit.cover, - mirrorMode: tile.local + mirrorMode: tile.local && !tile.screen ? lk.VideoViewMirrorMode.mirror : lk.VideoViewMirrorMode.off, ) @@ -349,37 +355,40 @@ class _Controls extends StatelessWidget { VoidCallback? onTap, { bool active = true, Color? colour, - }) => Column( - mainAxisSize: MainAxisSize.min, - children: [ - Material( - color: colour ?? (active ? Colors.white24 : Colors.white), - shape: const CircleBorder(), - child: InkWell( - customBorder: const CircleBorder(), - onTap: onTap, - child: SizedBox.square( - dimension: 60, - child: Icon( - icon, - color: colour != null || active ? Colors.white : Colors.black, + }) => Expanded( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Material( + color: colour ?? (active ? Colors.white24 : Colors.white), + shape: const CircleBorder(), + child: InkWell( + customBorder: const CircleBorder(), + onTap: onTap, + child: SizedBox.square( + dimension: 56, + child: Icon( + icon, + color: colour != null || active ? Colors.white : Colors.black, + ), ), ), ), - ), - const SizedBox(height: 6), - Text( - label, - style: const TextStyle(color: Colors.white70, fontSize: 12), - ), - ], + const SizedBox(height: 6), + Text( + label, + maxLines: 1, + overflow: TextOverflow.ellipsis, + style: const TextStyle(color: Colors.white70, fontSize: 12), + ), + ], + ), ); final connected = calls.phase == CallPhase.connected; return Padding( - padding: const EdgeInsets.fromLTRB(16, 12, 16, 20), + padding: const EdgeInsets.fromLTRB(8, 12, 8, 20), child: Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, children: [ round( calls.micOn ? Icons.mic : Icons.mic_off, @@ -399,6 +408,15 @@ class _Controls extends StatelessWidget { 'Flip', connected ? calls.flipCamera : null, ), + if (calls.canShareScreen) + round( + calls.screenSharing + ? Icons.stop_screen_share + : Icons.screen_share, + 'Screen', + connected ? calls.toggleScreenShare : null, + active: !calls.screenSharing, + ), round( Icons.call_end, 'Leave', diff --git a/apps/mobile/lib/screens/thread_list_screen.dart b/apps/mobile/lib/screens/thread_list_screen.dart index fd8642f..60bf648 100644 --- a/apps/mobile/lib/screens/thread_list_screen.dart +++ b/apps/mobile/lib/screens/thread_list_screen.dart @@ -3,13 +3,15 @@ import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:matrix/matrix.dart'; import '../matrix/client.dart'; +import '../matrix/thread_receipts.dart'; import '../state.dart'; import '../util/format.dart'; import '../widgets/avatar.dart'; import '../widgets/message_extras.dart'; import 'thread_screen.dart'; -/// A room's threads, the most recently active first (`/threads`). +/// A room's threads, the most recently active first (`/threads`); those +/// with replies you haven't read are bold, with a dot. class ThreadListScreen extends ConsumerStatefulWidget { const ThreadListScreen({super.key, required this.roomId}); final String roomId; @@ -41,6 +43,8 @@ class _ThreadListScreenState extends ConsumerState { @override Widget build(BuildContext context) { + // Receipts come with syncs: the unread marks follow them. + ref.watch(syncProvider); return Scaffold( appBar: AppBar(title: const Text('Threads')), body: RefreshIndicator( @@ -97,6 +101,7 @@ class _ThreadListScreenState extends ConsumerState { final when = latestTs == null ? root.originServerTs : DateTime.fromMillisecondsSinceEpoch(latestTs); + final unread = threadUnread(root); return ListTile( leading: Avatar( name: sender.calcDisplayname(), @@ -107,12 +112,23 @@ class _ThreadListScreenState extends ConsumerState { quoteText(root), maxLines: 2, overflow: TextOverflow.ellipsis, + style: unread + ? const TextStyle(fontWeight: FontWeight.w700) + : null, ), subtitle: Text( '${sender.calcDisplayname()} · ' '${count == 1 ? '1 reply' : '$count replies'} · ' '${formatListTime(when)}', ), + trailing: unread + ? Icon( + Icons.circle, + size: 10, + color: Theme.of(context).colorScheme.primary, + semanticLabel: 'Unread', + ) + : null, onTap: () => Navigator.of( context, ).push(ThreadScreen.route(widget.roomId, root.eventId)), diff --git a/apps/mobile/lib/screens/thread_screen.dart b/apps/mobile/lib/screens/thread_screen.dart index 07f6189..0ad720c 100644 --- a/apps/mobile/lib/screens/thread_screen.dart +++ b/apps/mobile/lib/screens/thread_screen.dart @@ -3,6 +3,7 @@ import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:matrix/matrix.dart'; import '../matrix/client.dart'; +import '../matrix/thread_receipts.dart'; import '../matrix/timeline.dart'; import '../state.dart'; import '../widgets/composer.dart'; @@ -15,6 +16,10 @@ import '../widgets/message.dart'; /// (which receives new replies, their sending status, edits and reactions /// like any event) and adds the older replies it doesn't have from /// `/relations`. +/// +/// While it is on screen and the app in front, it marks the thread read +/// with a receipt for the thread alone; "Read" under your replies comes +/// from the others' receipts in the thread. class ThreadScreen extends ConsumerStatefulWidget { const ThreadScreen({super.key, required this.roomId, required this.rootId}); final String roomId; @@ -29,7 +34,8 @@ class ThreadScreen extends ConsumerStatefulWidget { ConsumerState createState() => _ThreadScreenState(); } -class _ThreadScreenState extends ConsumerState { +class _ThreadScreenState extends ConsumerState + with WidgetsBindingObserver { Timeline? _timeline; Event? _root; Object? _error; @@ -42,17 +48,22 @@ class _ThreadScreenState extends ConsumerState { Event? _replyTo; final _scroll = ScrollController(); + /// The reply we last sent a receipt for, until the server echoes it. + String? _receiptSent; + Room get _room => ref.read(clientProvider).getRoomById(widget.roomId)!; @override void initState() { super.initState(); + WidgetsBinding.instance.addObserver(this); _scroll.addListener(_onScroll); _load(); } @override void dispose() { + WidgetsBinding.instance.removeObserver(this); _timeline?.cancelSubscriptions(); _scroll.dispose(); super.dispose(); @@ -64,6 +75,7 @@ class _ThreadScreenState extends ConsumerState { final timeline = await room.getTimeline( onUpdate: () { if (mounted) setState(() {}); + _markRead(); }, ); final root = @@ -78,11 +90,38 @@ class _ThreadScreenState extends ConsumerState { _root = root; }); await _loadOlder(); + _markRead(); } catch (e) { if (mounted) setState(() => _error = e); } } + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (state == AppLifecycleState.resumed) _markRead(); + } + + /// Sends a receipt in the thread for its newest reply, while the thread + /// is on screen and the app is in front. + void _markRead() { + if (!mounted || _timeline == null) return; + if (WidgetsBinding.instance.lifecycleState != AppLifecycleState.resumed) { + return; + } + final room = _room; + if (room.membership != Membership.join) return; + final newest = _replies().where((e) => e.status.isSynced).firstOrNull; + if (newest == null || newest.eventId == _receiptSent) return; + if (ownThreadReceipt(room, widget.rootId)?.eventId == newest.eventId) { + return; + } + _receiptSent = newest.eventId; + markThreadRead(room, widget.rootId, newest.eventId).catchError((Object e) { + Logs().w('Could not send a thread read receipt', e); + _receiptSent = null; + }); + } + Future _decrypt(Event event) async { final encryption = event.room.client.encryption; if (event.type != EventTypes.Encrypted || encryption == null) return event; @@ -171,6 +210,7 @@ class _ThreadScreenState extends ConsumerState { } else { final replies = _replies(); final entries = [...replies, ?root]; + final readUpTo = _readByOthersIndex(entries, room, widget.rootId); final items = projectTimeline([ for (final e in entries) ( @@ -217,7 +257,7 @@ class _ThreadScreenState extends ConsumerState { timeline: timeline, firstInGroup: firstInGroup, lastInGroup: lastInGroup, - readByOthers: false, + readByOthers: readUpTo != null && index >= readUpTo, onReply: (e) => setState(() => _replyTo = e), ), if (entries[index].eventId == widget.rootId) @@ -266,6 +306,21 @@ class _ThreadScreenState extends ConsumerState { } } +/// The index (newest first) of the newest entry someone else has read, +/// by their receipt in the thread or an unthreaded one, so your replies +/// from there back say "Read". +int? _readByOthersIndex(List entries, Room room, String rootId) { + for (var i = 0; i < entries.length; i++) { + final e = entries[i]; + if (!e.status.isSynced) continue; + if (threadReaders(room, rootId, e.eventId).isNotEmpty || + e.receipts.any((r) => r.user.id != room.client.userID)) { + return i; + } + } + return null; +} + class _ThreadDivider extends StatelessWidget { const _ThreadDivider(); diff --git a/apps/mobile/lib/widgets/message_extras.dart b/apps/mobile/lib/widgets/message_extras.dart index a1ca83f..0038ce5 100644 --- a/apps/mobile/lib/widgets/message_extras.dart +++ b/apps/mobile/lib/widgets/message_extras.dart @@ -2,6 +2,7 @@ import 'package:flutter/material.dart'; import 'package:matrix/matrix.dart'; import '../matrix/client.dart'; +import '../matrix/thread_receipts.dart'; import '../theme.dart'; import 'avatar.dart'; @@ -304,7 +305,8 @@ class QuoteBlock extends StatelessWidget { } } -/// Under a message that starts a thread: how many replies, opening it. +/// Under a message that starts a thread: how many replies, opening it; +/// a dot when it has replies you haven't read. class ThreadSummary extends StatelessWidget { const ThreadSummary({ super.key, @@ -340,6 +342,15 @@ class ThreadSummary extends StatelessWidget { fontWeight: FontWeight.w600, ), ), + if (threadUnread(event, timeline)) ...[ + const SizedBox(width: 6), + Icon( + Icons.circle, + size: 8, + color: c.tealText, + semanticLabel: 'Unread', + ), + ], ], ), ),