diff --git a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiSdkBridge.kt b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiSdkBridge.kt index f6f9fd9..317a797 100644 --- a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiSdkBridge.kt +++ b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiSdkBridge.kt @@ -117,6 +117,8 @@ class DjiSdkBridge( methodChannel.setMethodCallHandler(this) eventChannel.setStreamHandler(this) ctx.hub.start() + // The live view's picture size, for the touch layers' letterbox maths. + DjiVideoView.onVideoSize = { w, h -> emit(mapOf("type" to "videoSize", "width" to w, "height" to h)) } } // ── MethodChannel ────────────────────────────────────────────────────────── diff --git a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiVideoView.kt b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiVideoView.kt index dcf3262..5f25a53 100644 --- a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiVideoView.kt +++ b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/DjiVideoView.kt @@ -1,9 +1,12 @@ package com.pilotvault.flyapp import android.content.Context +import android.graphics.Color import android.graphics.SurfaceTexture +import android.view.Gravity import android.view.TextureView import android.view.View +import android.widget.FrameLayout import dji.sdk.camera.VideoFeeder import dji.sdk.codec.DJICodecManager import io.flutter.plugin.common.StandardMessageCodec @@ -12,12 +15,18 @@ import io.flutter.plugin.platform.PlatformViewFactory /** * Flutter [PlatformView] that renders the DJI product's live H.264 primary video - * feed as a full-bleed background for the Flight Control HUD. + * feed as the background of the Flight Control HUD. * * Pipeline: a [TextureView]'s [SurfaceTexture] is handed to a [DJICodecManager] * (hardware decoder). [VideoFeeder]'s primary-feed data listener pushes raw * frames straight into the decoder, which renders onto the surface. * + * The decoder stretches the picture over whatever surface it gets, so the + * texture sits centred in a black frame, sized to the feed's own aspect ratio + * once the decoder reports it (letterboxed, as GO 4 shows it). The size also + * goes to [onVideoSize] — `DjiSdkBridge` publishes it as a `videoSize` event, + * so the Dart touch layers can map taps into the picture rather than the bars. + * * Registered under [VIEW_TYPE] in `MainActivity.configureFlutterEngine`; embedded * on the Dart side by `AndroidView(viewType: 'dji_msdk/video')`. * @@ -30,30 +39,79 @@ class DjiVideoView(context: Context) : PlatformView, TextureView.SurfaceTextureL companion object { const val VIEW_TYPE = "dji_msdk/video" + + /** Called on the main thread with the feed's width and height in pixels. */ + @Volatile + var onVideoSize: ((Int, Int) -> Unit)? = null } + private val frame = FrameLayout(context).also { it.setBackgroundColor(Color.BLACK) } + private val textureView = TextureView(context).also { it.surfaceTextureListener = this + frame.addView(it, FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT, Gravity.CENTER)) } private var codecManager: DJICodecManager? = null + private var videoWidth = 0 + private var videoHeight = 0 + + init { + // Re-fit when the platform view itself is resized (the map / video swap). + frame.addOnLayoutChangeListener { _, l, t, r, b, ol, ot, or_, ob -> + if (r - l != or_ - ol || b - t != ob - ot) fit() + } + } // Pushes raw H.264 frames from the SDK straight into the hardware decoder. private val videoDataListener = VideoFeeder.VideoDataListener { data, size -> codecManager?.sendDataToDecoder(data, size) } - override fun getView(): View = textureView + override fun getView(): View = frame override fun dispose() { teardown() } + /** Sizes the texture to the feed's aspect ratio inside the frame. */ + private fun fit() { + val fw = frame.width + val fh = frame.height + if (fw <= 0 || fh <= 0) return + val lp = textureView.layoutParams as FrameLayout.LayoutParams + if (videoWidth <= 0 || videoHeight <= 0) { + lp.width = FrameLayout.LayoutParams.MATCH_PARENT + lp.height = FrameLayout.LayoutParams.MATCH_PARENT + } else { + val aspect = videoWidth.toFloat() / videoHeight + if (fw.toFloat() / fh > aspect) { + lp.height = fh + lp.width = (fh * aspect).toInt() + } else { + lp.width = fw + lp.height = (fw / aspect).toInt() + } + } + lp.gravity = Gravity.CENTER + textureView.layoutParams = lp + } + // ── TextureView.SurfaceTextureListener ─────────────────────────────────── override fun onSurfaceTextureAvailable(surface: SurfaceTexture, width: Int, height: Int) { if (codecManager == null) { - codecManager = DJICodecManager(textureView.context, surface, width, height) + codecManager = DJICodecManager(textureView.context, surface, width, height).also { codec -> + codec.setOnVideoSizeChangedListener { w, h -> + frame.post { + if (w == videoWidth && h == videoHeight) return@post + videoWidth = w + videoHeight = h + fit() + onVideoSize?.invoke(w, h) + } + } + } } VideoFeeder.getInstance()?.primaryVideoFeed?.addVideoDataListener(videoDataListener) } diff --git a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/FlightAssistantBridge.kt b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/FlightAssistantBridge.kt index b76b756..8178de7 100644 --- a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/FlightAssistantBridge.kt +++ b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/FlightAssistantBridge.kt @@ -17,6 +17,7 @@ import io.flutter.plugin.common.MethodChannel * ``` * * Aircraft without obstacle sensing never call back, so `vision` stays absent. + * The palm-control state (`PalmControlState` name) goes out as `palmControl`. */ class FlightAssistantBridge(private val ctx: BridgeCtx) : SubBridge { @@ -40,6 +41,10 @@ class FlightAssistantBridge(private val ctx: BridgeCtx) : SubBridge { ctx.hub.putFast(mapOf("vision" to sensors.toMap())) } } + // Palm control (Spark's advanced gestures): only aircraft that have it call back. + assistant.setPalmControlStateCallback { state -> + ctx.mainHandler.post { ctx.hub.putFast(mapOf("palmControl" to state?.name)) } + } } override fun reset() { diff --git a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/MissionBridge.kt b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/MissionBridge.kt index fe2b72d..d0aa9bb 100644 --- a/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/MissionBridge.kt +++ b/Fly App/android/app/src/main/kotlin/com/pilotvault/flyapp/MissionBridge.kt @@ -22,6 +22,7 @@ import dji.common.mission.waypoint.WaypointMissionHeadingMode import dji.common.mission.waypoint.WaypointMissionUploadEvent import dji.common.mission.waypoint.WaypointTurnMode import dji.common.model.LocationCoordinate2D +import dji.common.util.CommonCallbacks import dji.sdk.base.BaseProduct import dji.sdk.mission.activetrack.ActiveTrackMissionOperatorListener import dji.sdk.mission.activetrack.ActiveTrackOperator @@ -97,8 +98,16 @@ class MissionBridge(private val ctx: BridgeCtx) : SubBridge { "index" to t.targetIndex, "percent" to t.completionPercentage, "reason" to t.reason?.name, + // Subjects the aircraft found itself (auto-sensing), to tap instead of drawing. + "subjects" to t.autoSensedSubjects.orEmpty().filterNotNull().map { s -> + mapOf("index" to s.index, "state" to s.state?.name, "type" to s.targetType?.name, "rect" to rect(s.targetRect)) + }, ) } + trackOp()?.let { + m["autoSensingSupported"] = it.isAutoSensingSupported + m["autoSensing"] = it.isAutoSensingEnabled + } event.error?.let { m["error"] = it.description } ctx.emit(m) } @@ -180,7 +189,7 @@ class MissionBridge(private val ctx: BridgeCtx) : SubBridge { op.addListener(trackListener) listenedTrack = op } - ctx.emit(mapOf("type" to "mission", "operator" to "activeTrack", "state" to op.currentState?.name)) + emitTrackFlags(op) } tapFlyOp()?.let { op -> if (listenedTapFly !== op) { @@ -225,6 +234,28 @@ class MissionBridge(private val ctx: BridgeCtx) : SubBridge { "setActiveTrackRetreat" -> track(result) { it.setRetreatEnabled(call.argument("on") ?: false, ctx.completion(result)) } + "getActiveTrackRetreat" -> track(result) { op -> + op.getRetreatEnabled(object : CommonCallbacks.CompletionCallbackWith { + override fun onSuccess(v: Boolean) { ctx.mainHandler.post { result.success(v) } } + override fun onFailure(e: DJIError) { ctx.mainHandler.post { result.error("DJI_ERROR", e.description, null) } } + }) + } + "setActiveTrackAutoSensing" -> track(result) { op -> + val on = call.argument("on") ?: false + val done = ctx.completion(result) + val cb = CommonCallbacks.CompletionCallback { e -> + done.onResult(e) + emitTrackFlags(op) + } + if (on) op.enableAutoSensing(cb) else op.disableAutoSensing(cb) + } + "startActiveTrackSubject" -> track(result) { op -> + // Follows one of the auto-sensed subjects by its index. + val mission = ActiveTrackMission() + mission.targetIndex = call.argument("index")?.toInt() ?: 0 + mission.mode = enumArg(call, "mode", ActiveTrackMode.TRACE) + op.startAutoSensingMission(mission, ctx.completion(result)) + } "startQuickShot" -> startQuickShot(call, result) "startTapFly" -> startTapFly(call, result) "stopTapFly" -> tapFly(result) { it.stopMission(ctx.completion(result)) } @@ -247,6 +278,19 @@ class MissionBridge(private val ctx: BridgeCtx) : SubBridge { block(op) } + /** The operator's state plus whether auto-sensing exists and is on. */ + private fun emitTrackFlags(op: ActiveTrackOperator) { + ctx.emit( + mapOf( + "type" to "mission", + "operator" to "activeTrack", + "state" to op.currentState?.name, + "autoSensingSupported" to op.isAutoSensingSupported, + "autoSensing" to op.isAutoSensingEnabled, + ), + ) + } + private inline fun tapFly(result: MethodChannel.Result, block: (TapFlyMissionOperator) -> Unit) { val op = tapFlyOp() ?: run { ctx.fail(result, "TapFly unavailable"); return } block(op) diff --git a/Fly App/lib/dji_service.dart b/Fly App/lib/dji_service.dart index b5679b5..9d17202 100644 --- a/Fly App/lib/dji_service.dart +++ b/Fly App/lib/dji_service.dart @@ -309,6 +309,15 @@ class DjiService { /// Lets Trace fly backwards ahead of a subject walking towards the aircraft. Future setActiveTrackRetreat(bool on) => _methods.invokeMethod('setActiveTrackRetreat', {'on': on}); + Future getActiveTrackRetreat() => _methods.invokeMethod('getActiveTrackRetreat'); + + /// Lets the aircraft look for subjects itself; they arrive in `tracking.subjects`. + Future setActiveTrackAutoSensing(bool on) => + _methods.invokeMethod('setActiveTrackAutoSensing', {'on': on}); + + /// Follows the auto-sensed subject with operator index [index]. + Future startActiveTrackSubject(int index, {String mode = 'TRACE'}) => + _methods.invokeMethod('startActiveTrackSubject', {'index': index, 'mode': mode}); // ── TapFly ────────────────────────────────────────────────────────────────── /// Flies towards the point [x], [y] (0..1 across the live view). diff --git a/Fly App/lib/flight_model.dart b/Fly App/lib/flight_model.dart index c887c03..677926e 100644 --- a/Fly App/lib/flight_model.dart +++ b/Fly App/lib/flight_model.dart @@ -1,3 +1,5 @@ +import 'dart:ui' show Size; + import 'package:flutter/foundation.dart'; import 'fly/product_profile.dart'; @@ -132,6 +134,10 @@ class FlightModel extends ChangeNotifier { /// The calibration wizards' progress (plan P6b). final CalibrationState calibration = CalibrationState(); + /// The live feed's picture size in pixels, once the decoder reports it: the + /// native view letterboxes to it, and the touch layers map taps into it. + final ValueNotifier videoSize = ValueNotifier(null); + late final List _subs = [ aircraft, battery, links, camera, gimbal, vision, alerts, mission, keys, calibration, // ]; @@ -254,6 +260,7 @@ class FlightModel extends ChangeNotifier { sub.removeListener(_onSub); sub.dispose(); } + videoSize.dispose(); super.dispose(); } } diff --git a/Fly App/lib/fly/hud_features.dart b/Fly App/lib/fly/hud_features.dart index 2200744..8d49ee7 100644 --- a/Fly App/lib/fly/hud_features.dart +++ b/Fly App/lib/fly/hud_features.dart @@ -105,6 +105,9 @@ abstract final class HudFeatures { 'settings.obstacle', 'Obstacle avoidance settings', 'Settings', (ProductProfile p) => p.hasObstacleSensing); /// Transmission rows address the link's own key class, so each set is gated /// on the link type. + /// Gesture and palm control ride on ActiveTrack's vision. + static final HudFeature settingsGestures = HudFeature('settings.gestures', 'Gesture & palm control', 'Settings', + (ProductProfile p) => p.supportsMode(IntelligentMode.activeTrack)); static final HudFeature linkOcuSync = HudFeature( 'settings.ocusync', 'OcuSync transmission settings', 'Settings', (ProductProfile p) => p.link == LinkType.ocusync); static final HudFeature linkLightbridge = HudFeature('settings.lightbridge', 'Lightbridge transmission settings', 'Settings', @@ -175,6 +178,7 @@ abstract final class HudFeatures { simulator, settingsVision, settingsObstacle, + settingsGestures, linkOcuSync, linkLightbridge, linkWifi, diff --git a/Fly App/lib/fly/modes/vision_modes.dart b/Fly App/lib/fly/modes/vision_modes.dart index feed09f..5dbb24f 100644 --- a/Fly App/lib/fly/modes/vision_modes.dart +++ b/Fly App/lib/fly/modes/vision_modes.dart @@ -12,9 +12,27 @@ class VisionSession extends ChangeNotifier { /// A box drawn (or tapped) on the video, 0..1, waiting for GO. Rect? pendingBox; + /// When [pendingBox] is a subject the aircraft found itself: its index. + int? pendingSubject; + /// A TapFly target, 0..1, waiting for GO. Offset? pendingPoint; + /// Trace may fly backwards ahead of a subject coming towards it; null until read. + bool? retreat; + + void setRetreat(bool? on) { + retreat = on; + notifyListeners(); + } + + /// Picks an auto-sensed subject: its box with its index. + void pickSubject(int index, Rect box) { + pendingBox = box; + pendingSubject = index; + notifyListeners(); + } + void setTrackMode(TrackMode m) { trackMode = m; notifyListeners(); @@ -32,6 +50,7 @@ class VisionSession extends ChangeNotifier { void setPendingBox(Rect? r) { pendingBox = r; + pendingSubject = null; notifyListeners(); } @@ -43,6 +62,7 @@ class VisionSession extends ChangeNotifier { void clearPending() { if (pendingBox == null && pendingPoint == null) return; pendingBox = null; + pendingSubject = null; pendingPoint = null; notifyListeners(); } @@ -84,40 +104,70 @@ enum TapFlyDirection { final String sdk; } -/// The live-view geometry the vision modes share: screen points to the -/// operators' 0..1 coordinates. The feed is assumed to fill [view] (the same -/// assumption as tap-to-focus; letterboxing is ignored). +/// The live-view geometry the touch layers share: screen points to the +/// operators' 0..1 picture coordinates. +/// +/// The native view letterboxes the feed to its own aspect ratio, so the +/// picture is [contentRect] — centred in the view, with black bars on two +/// sides. Every mapping goes through that rect; a touch on a bar is no touch. abstract final class VisionGeometry { /// A drawn box smaller than this (px, either side) is taken as a tap. static const double minDrag = 24; - /// The box a tap makes, as a fraction of the view: a person-sized window. + /// The box a tap makes, as a fraction of the picture: a person-sized window. static const Size tapBox = Size(0.12, 0.22); - /// The 0..1 box between two corners of a drag, clipped to the view; null - /// when it is too small to be a deliberate box. - static Rect? dragBox(Offset a, Offset b, Size view) { + /// Where the picture of size [video] sits inside [view] (aspect fit, + /// centred); the whole view while the feed size is unknown. + static Rect contentRect(Size view, Size? video) { + if (video == null || video.isEmpty || view.isEmpty) return Offset.zero & view; + final double aspect = video.width / video.height; + final double w, h; + if (view.width / view.height > aspect) { + h = view.height; + w = h * aspect; + } else { + w = view.width; + h = w / aspect; + } + return Rect.fromLTWH((view.width - w) / 2, (view.height - h) / 2, w, h); + } + + /// The 0..1 box between two corners of a drag, clipped to the picture; null + /// when it is too small to be a deliberate box, or misses the picture. + static Rect? dragBox(Offset a, Offset b, Rect content) { if ((a.dx - b.dx).abs() < minDrag || (a.dy - b.dy).abs() < minDrag) return null; - final Rect r = Rect.fromPoints(a, b); + final Rect r = Rect.fromPoints(a, b).intersect(content); + if (r.width < minDrag || r.height < minDrag) return null; return Rect.fromLTRB( - (r.left / view.width).clamp(0.0, 1.0), - (r.top / view.height).clamp(0.0, 1.0), - (r.right / view.width).clamp(0.0, 1.0), - (r.bottom / view.height).clamp(0.0, 1.0), + (r.left - content.left) / content.width, + (r.top - content.top) / content.height, + (r.right - content.left) / content.width, + (r.bottom - content.top) / content.height, ); } - /// A [tapBox] centred on a tap, kept inside the view. - static Rect tapBoxAt(Offset p, Size view) { - final double cx = (p.dx / view.width).clamp(tapBox.width / 2, 1 - tapBox.width / 2); - final double cy = (p.dy / view.height).clamp(tapBox.height / 2, 1 - tapBox.height / 2); + /// A [tapBox] centred on a tap, kept inside the picture; null on a bar. + static Rect? tapBoxAt(Offset p, Rect content) { + final Offset? n = normalise(p, content); + if (n == null) return null; + final double cx = n.dx.clamp(tapBox.width / 2, 1 - tapBox.width / 2); + final double cy = n.dy.clamp(tapBox.height / 2, 1 - tapBox.height / 2); return Rect.fromCenter(center: Offset(cx, cy), width: tapBox.width, height: tapBox.height); } - static Offset normalise(Offset p, Size view) => - Offset((p.dx / view.width).clamp(0.0, 1.0), (p.dy / view.height).clamp(0.0, 1.0)); + /// [p] as 0..1 of the picture; null when it falls on a bar. + static Offset? normalise(Offset p, Rect content) { + if (!content.contains(p)) return null; + return Offset((p.dx - content.left) / content.width, (p.dy - content.top) / content.height); + } - static Rect toScreen(Rect r, Size view) => Rect.fromLTRB(r.left * view.width, r.top * view.height, r.right * view.width, r.bottom * view.height); + static Rect toScreen(Rect r, Rect content) => Rect.fromLTRB( + content.left + r.left * content.width, + content.top + r.top * content.height, + content.left + r.right * content.width, + content.top + r.bottom * content.height, + ); - static Offset pointToScreen(Offset p, Size view) => Offset(p.dx * view.width, p.dy * view.height); + static Offset pointToScreen(Offset p, Rect content) => Offset(content.left + p.dx * content.width, content.top + p.dy * content.height); } diff --git a/Fly App/lib/fly/safety_gate.dart b/Fly App/lib/fly/safety_gate.dart index 1e64350..54853c2 100644 --- a/Fly App/lib/fly/safety_gate.dart +++ b/Fly App/lib/fly/safety_gate.dart @@ -496,6 +496,38 @@ class SafetyGate { await dji.startActiveTrack(r.left, r.top, r.width, r.height, mode: mode.sdk); } + /// Re-checks and follows the auto-sensed subject [index]. + Future startActiveTrackSubject(int index, TrackMode mode) async { + final List checks = checkActiveTrack(mode); + if (checks.any((GateCheck c) => c.blocks)) throw MissionRefused(checks); + await dji.startActiveTrackSubject(index, mode: mode.sdk); + } + + /// Turns the aircraft's own subject detection on or off (not while following). + Future setAutoSensing(bool on) async { + if (!model.connected) throw const MissionRefused([GateCheck.block('No aircraft connected.')]); + if (!model.mission.activeTrack.autoSensingSupported) { + throw const MissionRefused([GateCheck.block('This aircraft cannot detect subjects itself.')]); + } + if (model.mission.activeTrack.active) throw const MissionRefused([GateCheck.block('Stop tracking first.')]); + await dji.setActiveTrackAutoSensing(on); + } + + /// Trace's "fly backwards ahead of a subject walking towards the aircraft". + Future setTrackRetreat(bool on) async { + if (!model.connected) throw const MissionRefused([GateCheck.block('No aircraft connected.')]); + if (on && !model.profile.hasObstacleSensing) { + throw const MissionRefused([GateCheck.block('Flying backwards needs obstacle sensing.')]); + } + await dji.setActiveTrackRetreat(on); + } + + /// In TapFly Free, turns the aircraft back to face where it is flying. + Future resetTapFlyHeading() async { + if (!model.mission.tapFly.active) throw const MissionRefused([GateCheck.block('TapFly is not flying.')]); + await dji.resetTapFlyHeading(); + } + /// Answers the operator's "is this the subject?". Future confirmActiveTrack({required bool yes}) async { if (!model.mission.activeTrack.waitingForConfirmation && !model.mission.activeTrack.cannotConfirm) return; diff --git a/Fly App/lib/fly/settings/settings_tabs.dart b/Fly App/lib/fly/settings/settings_tabs.dart index 9445958..5588333 100644 --- a/Fly App/lib/fly/settings/settings_tabs.dart +++ b/Fly App/lib/fly/settings/settings_tabs.dart @@ -43,6 +43,8 @@ abstract final class SettingsKeys { static const DjiKey landingProtection = DjiKey.flightAssistant('LANDING_PROTECTION_ENABLED'); static const DjiKey precisionLanding = DjiKey.flightAssistant('PRECISION_LANDING_ENABLED'); static const DjiKey bottomLight = DjiKey.flightAssistant('BOTTOM_AUXILIARY_LIGHT_MODE'); + static const DjiKey gestureMode = DjiKey.flightAssistant('ACTIVE_TRACK_GESTURE_MODE_ENABLED'); + static const DjiKey palmControl = DjiKey.flightAssistant('ADVANCED_GESTURE_CONTROL_ENABLED'); // Remote controller static const DjiKey stickMode = DjiKey.remoteController('AIRCRAFT_MAPPING_STYLE'); @@ -256,6 +258,22 @@ abstract final class SettingsTabs { key: SettingsKeys.bottomLight, options: <(String, String)>[('AUTO', 'Auto'), ('ON', 'On'), ('OFF', 'Off')], ), + // Rows hide on aircraft whose flight assistant does not support the key. + GroupSpec('GESTURES', requires: HudFeatures.settingsGestures), + KeySwitchSpec( + id: 'gestureMode', + label: 'Gesture mode', + hint: 'Raise your arms to be tracked; make a frame with your fingers for a photo', + key: SettingsKeys.gestureMode, + requires: HudFeatures.settingsGestures, + ), + KeySwitchSpec( + id: 'palmControl', + label: 'Palm control', + hint: 'Guide the aircraft with your palm, a couple of metres away', + key: SettingsKeys.palmControl, + requires: HudFeatures.settingsGestures, + ), ]; // ── Remote controller ─────────────────────────────────────────────────────── diff --git a/Fly App/lib/fly/state/mission_state.dart b/Fly App/lib/fly/state/mission_state.dart index 03d8a63..67f6236 100644 --- a/Fly App/lib/fly/state/mission_state.dart +++ b/Fly App/lib/fly/state/mission_state.dart @@ -91,7 +91,21 @@ class WaypointProgress { /// The ActiveTrack operator as last reported (plan P7b). @immutable class ActiveTrackProgress { - const ActiveTrackProgress({this.state = 'UNKNOWN', this.targetState, this.targetType, this.rect, this.reason}); + const ActiveTrackProgress({ + this.state = 'UNKNOWN', + this.targetState, + this.targetType, + this.rect, + this.reason, + this.subjects = const [], + this.autoSensingSupported = false, + this.autoSensing = false, + }); + + /// Subjects the aircraft found itself (auto-sensing), to tap instead of drawing. + final List subjects; + final bool autoSensingSupported; + final bool autoSensing; factory ActiveTrackProgress.fromMap(Map e, ActiveTrackProgress previous) { final Map? t = asMap(e['tracking']); @@ -105,6 +119,15 @@ class ActiveTrackProgress { targetType: t == null ? (keep ? previous.targetType : null) : asString(t['targetType']), rect: t == null ? (keep ? previous.rect : null) : _rect(r), reason: t == null ? (keep ? previous.reason : null) : asString(t['reason']), + // Sensed subjects only mean something while auto-sensing looks for them. + subjects: t == null + ? (state == 'AUTO_SENSING' ? previous.subjects : const []) + : [ + for (final Map s in asMapList(t['subjects'])) + if (_rect(asMap(s['rect'])) case final Rect box) SensedSubject(asInt(s['index']) ?? 0, box, asString(s['type'])), + ], + autoSensingSupported: asBool(e['autoSensingSupported']) ?? previous.autoSensingSupported, + autoSensing: asBool(e['autoSensing']) ?? previous.autoSensing, ); } @@ -156,6 +179,7 @@ class ActiveTrackProgress { 'PERFORMING_QUICK_SHOT' => 'QuickShot', 'CANNOT_START' => 'Cannot start', 'NOT_SUPPORT' => 'Not supported', + 'AUTO_SENSING' => subjects.isEmpty ? 'Looking for subjects' : 'Tap a subject', _ => 'Draw a box', }; @@ -179,10 +203,29 @@ class ActiveTrackProgress { other.targetState == targetState && other.targetType == targetType && other.rect == rect && - other.reason == reason; + other.reason == reason && + listEquals(other.subjects, subjects) && + other.autoSensingSupported == autoSensingSupported && + other.autoSensing == autoSensing; @override - int get hashCode => Object.hash(state, targetState, targetType, rect, reason); + int get hashCode => Object.hash(state, targetState, targetType, rect, reason, Object.hashAll(subjects), autoSensingSupported, autoSensing); +} + +/// One subject auto-sensing found: its operator index and box (0..1 of the picture). +@immutable +class SensedSubject { + const SensedSubject(this.index, this.rect, this.type); + + final int index; + final Rect rect; + final String? type; + + @override + bool operator ==(Object other) => other is SensedSubject && other.index == index && other.rect == rect && other.type == type; + + @override + int get hashCode => Object.hash(index, rect, type); } /// The TapFly operator as last reported (plan P7b). diff --git a/Fly App/lib/fly/state/vision_state.dart b/Fly App/lib/fly/state/vision_state.dart index 002d821..45c1eba 100644 --- a/Fly App/lib/fly/state/vision_state.dart +++ b/Fly App/lib/fly/state/vision_state.dart @@ -86,9 +86,30 @@ class VisionState extends ChangeNotifier { notifyListeners(); } + /// MSDK `PalmControlState` (Spark's palm control), from the tick's `palmControl`. + String? palmControl; + + /// The aircraft is reading or following the pilot's palm. + bool get palmActive => const {'INITIALIZING', 'RECOGNIZING_PALM', 'STARTING_FOLLOW', 'FOLLOWING', 'ADJUSTING_POSITION'}.contains(palmControl); + + String get palmLabel => switch (palmControl) { + 'INITIALIZING' || 'RECOGNIZING_PALM' => 'Looking for your palm', + 'STARTING_FOLLOW' => 'Palm found', + 'FOLLOWING' => 'Following your palm', + 'ADJUSTING_POSITION' => 'Adjusting position', + _ => 'Off', + }; + + void applyPalm(String? state) { + if (state == palmControl) return; + palmControl = state; + notifyListeners(); + } + void clear() { sensors = const {}; updatedAt = null; + palmControl = null; notifyListeners(); } } diff --git a/Fly App/lib/fly/telemetry_reducer.dart b/Fly App/lib/fly/telemetry_reducer.dart index 98db415..726c89b 100644 --- a/Fly App/lib/fly/telemetry_reducer.dart +++ b/Fly App/lib/fly/telemetry_reducer.dart @@ -1,3 +1,5 @@ +import 'dart:ui' show Size; + import '../flight_model.dart'; import 'state/parse.dart'; @@ -20,7 +22,7 @@ class TelemetryReducer { final Map _lastGroups = {}; - static const List _groups = ['battery', 'links', 'rc', 'storage', 'ssd', 'thermal', 'cameraSettings', 'gimbal', 'gimbalSettings', 'vision', 'calibration']; + static const List _groups = ['battery', 'links', 'rc', 'storage', 'ssd', 'thermal', 'cameraSettings', 'gimbal', 'gimbalSettings', 'vision', 'calibration', 'palmControl']; /// Applies [event] if it is one of the telemetry types; returns whether it was. bool apply(Map event) { @@ -48,6 +50,10 @@ class TelemetryReducer { case 'mission': model.mission.apply(event); return true; + case 'videoSize': + final int? w = asInt(event['width']), h = asInt(event['height']); + if (w != null && h != null && w > 0 && h > 0) model.videoSize.value = Size(w.toDouble(), h.toDouble()); + return true; } return false; } @@ -90,5 +96,6 @@ class TelemetryReducer { if (changed.contains('gimbalSettings')) model.gimbal.applySettings(asMap(t['gimbalSettings'])); if (changed.contains('vision')) model.vision.apply(asMap(t['vision'])); if (changed.contains('calibration')) model.calibration.apply(asMap(t['calibration'])); + if (changed.contains('palmControl')) model.vision.applyPalm(asString(t['palmControl'])); } } diff --git a/Fly App/lib/ui/fly/fly_screen.dart b/Fly App/lib/ui/fly/fly_screen.dart index eaecc5b..741fb6c 100644 --- a/Fly App/lib/ui/fly/fly_screen.dart +++ b/Fly App/lib/ui/fly/fly_screen.dart @@ -179,6 +179,13 @@ class _FlyScreenState extends State { _mode = m; _mapBig = false; }); + if (m == IntelligentMode.activeTrack && _m.connected) { + try { + _vision.setRetreat(await _dji.getActiveTrackRetreat()); + } catch (_) { + _vision.setRetreat(null); // not reported: the chip starts off + } + } default: break; // the picker only returns built modes } @@ -201,7 +208,7 @@ class _FlyScreenState extends State { if (mounted) setState(() => _mode = null); } - void _goTrack(BuildContext context, Rect box) { + void _goTrack(BuildContext context, Rect box, {int? subject}) { final TrackMode mode = _vision.trackMode; startVisionMode( context, @@ -210,7 +217,8 @@ class _FlyScreenState extends State { description: 'The aircraft follows the boxed subject. Stop, the remote\'s pause button, or the sticks take back control.', checks: _gate.checkActiveTrack(mode), start: () async { - await _gate.startActiveTrack(box, mode); + // A subject the aircraft sensed itself is started by its index, not a drawn box. + await (subject != null ? _gate.startActiveTrackSubject(subject, mode) : _gate.startActiveTrack(box, mode)); _vision.setPendingBox(null); }, onError: _error, @@ -310,20 +318,21 @@ class _FlyScreenState extends State { // the swap moves the native video view instead of recreating it. if (!_mapBig) ...[ Positioned.fromRect(key: const ValueKey('video-slot'), rect: full, child: video), - const Positioned.fill(child: VisualOverlays()), + Positioned.fill(child: VisualOverlays(video: _m.videoSize)), // A vision mode owns every touch on the video: a box or a point. if (_mode == IntelligentMode.activeTrack) Positioned.fill( child: ActiveTrackLayer( mission: _m.mission, session: _vision, - onGo: (Rect box) => _goTrack(context, box), + video: _m.videoSize, + onGo: (Rect box, int? subject) => _goTrack(context, box, subject: subject), onConfirm: _confirmTrack, ), ) else if (_mode == IntelligentMode.tapFly) Positioned.fill( - child: TapFlyLayer(mission: _m.mission, session: _vision, onGo: (Offset p) => _goTapFly(context, p)), + child: TapFlyLayer(mission: _m.mission, session: _vision, video: _m.videoSize, onGo: (Offset p) => _goTapFly(context, p)), ) else ...[ // Under the camera touch layer, in the same gesture arena: drags aim @@ -349,7 +358,7 @@ class _FlyScreenState extends State { child: ListenableBuilder( listenable: _m, builder: (BuildContext context, _) => - CameraTouchLayer(camera: _m.camera, dji: _dji, connected: _m.connected, onError: _error), + CameraTouchLayer(camera: _m.camera, dji: _dji, connected: _m.connected, onError: _error, video: _m.videoSize), ), ), ), @@ -512,6 +521,26 @@ class _FlyScreenState extends State { bottom: 104, child: Column(mainAxisSize: MainAxisSize.min, crossAxisAlignment: CrossAxisAlignment.start, children: [ WaypointRunPanel(model: _m, editor: _waypoints, gate: _gate, flow: _flow, onError: _error), + // Palm control (Spark) flies the aircraft from the pilot's hand: say so. + ListenableBuilder( + listenable: _m.vision, + builder: (BuildContext context, _) => !_m.vision.palmActive + ? const SizedBox.shrink() + : Padding( + padding: const EdgeInsets.only(bottom: 6), + child: GlassPill( + key: const ValueKey('palm-control'), + color: Glass.pillStrong, + child: Row(mainAxisSize: MainAxisSize.min, children: [ + const PVIcon('user', size: 12, color: Glass.ink), + const SizedBox(width: 6), + Text('PALM CONTROL', style: HudText.label.copyWith(color: Glass.ink, fontWeight: FontWeight.w700)), + const SizedBox(width: 6), + Text(_m.vision.palmLabel.toUpperCase(), style: HudText.label.copyWith(color: HudColors.ready)), + ]), + ), + ), + ), if (_visionMode) Padding( padding: const EdgeInsets.only(bottom: 6), diff --git a/Fly App/lib/ui/fly/layers/video_layer.dart b/Fly App/lib/ui/fly/layers/video_layer.dart index e78f156..dafe195 100644 --- a/Fly App/lib/ui/fly/layers/video_layer.dart +++ b/Fly App/lib/ui/fly/layers/video_layer.dart @@ -1,6 +1,8 @@ +import 'package:flutter/foundation.dart' show ValueListenable; import 'package:flutter/material.dart'; import '../../../fly/fly_prefs.dart'; +import '../../../fly/modes/vision_modes.dart' show VisionGeometry; import '../../dji_video_view.dart'; import '../../pv_icons.dart'; @@ -18,18 +20,34 @@ class VideoLayer extends StatelessWidget { /// The visual overlays drawn straight on the video: the grid and the centre /// point, as set in the camera menu (`FlyPrefs`). Never takes touches. +/// +/// The grid divides the picture, not the letterbox bars around it: [video] is +/// the feed's size, as the native view fits it. class VisualOverlays extends StatelessWidget { - const VisualOverlays({super.key}); + const VisualOverlays({super.key, this.video}); + + final ValueListenable? video; @override Widget build(BuildContext context) { + final Widget grid = ValueListenableBuilder( + valueListenable: FlyPrefs.grid, + builder: (BuildContext context, GridStyle g, _) => + g == GridStyle.off ? const SizedBox.shrink() : CustomPaint(painter: GridPainter(diagonals: g == GridStyle.diagonals)), + ); return IgnorePointer( child: Stack(fit: StackFit.expand, children: [ - ValueListenableBuilder( - valueListenable: FlyPrefs.grid, - builder: (BuildContext context, GridStyle g, _) => - g == GridStyle.off ? const SizedBox.shrink() : CustomPaint(painter: GridPainter(diagonals: g == GridStyle.diagonals)), - ), + if (video == null) + grid + else + LayoutBuilder( + builder: (BuildContext context, BoxConstraints c) => ValueListenableBuilder( + valueListenable: video!, + builder: (BuildContext context, Size? v, Widget? child) => + Stack(children: [Positioned.fromRect(rect: VisionGeometry.contentRect(c.biggest, v), child: child!)]), + child: grid, + ), + ), ValueListenableBuilder( valueListenable: FlyPrefs.centerPoint, builder: (BuildContext context, bool on, _) => on diff --git a/Fly App/lib/ui/fly/modes/vision/vision_layers.dart b/Fly App/lib/ui/fly/modes/vision/vision_layers.dart index 8414257..9cd458c 100644 --- a/Fly App/lib/ui/fly/modes/vision/vision_layers.dart +++ b/Fly App/lib/ui/fly/modes/vision/vision_layers.dart @@ -1,5 +1,6 @@ import 'dart:math' as math; +import 'package:flutter/foundation.dart' show ValueListenable; import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; @@ -18,11 +19,23 @@ import '../../hud_kit.dart'; /// Full-screen, in place of the camera and gimbal touch layers while the /// mode is open: every touch on the video means "this subject". class ActiveTrackLayer extends StatefulWidget { - const ActiveTrackLayer({super.key, required this.mission, required this.session, required this.onGo, required this.onConfirm}); + const ActiveTrackLayer({ + super.key, + required this.mission, + required this.session, + required this.video, + required this.onGo, + required this.onConfirm, + }); final MissionState mission; final VisionSession session; - final ValueChanged onGo; + + /// The feed's picture size, for the letterbox ([VisionGeometry.contentRect]). + final ValueListenable video; + + /// GO on a box; the subject index when it is one the aircraft sensed itself. + final void Function(Rect box, int? subject) onGo; final ValueChanged onConfirm; @override @@ -33,22 +46,36 @@ class _ActiveTrackLayerState extends State { Offset? _from; Offset? _to; + /// A tap on a sensed subject picks it; anywhere else on the picture makes a box. + void _tap(Offset p, Rect content, List subjects) { + for (final SensedSubject s in subjects) { + if (VisionGeometry.toScreen(s.rect, content).inflate(8).contains(p)) { + widget.session.pickSubject(s.index, s.rect); + return; + } + } + final Rect? box = VisionGeometry.tapBoxAt(p, content); + if (box != null) widget.session.setPendingBox(box); + } + @override Widget build(BuildContext context) { return LayoutBuilder(builder: (BuildContext context, BoxConstraints c) { - final Size view = c.biggest; + final Size full = c.biggest; return ListenableBuilder( - listenable: Listenable.merge([widget.mission, widget.session]), + listenable: Listenable.merge([widget.mission, widget.session, widget.video]), builder: (BuildContext context, _) { + final Rect view = VisionGeometry.contentRect(full, widget.video.value); final ActiveTrackProgress t = widget.mission.activeTrack; final bool tracking = t.active; final Rect? pending = tracking ? null : widget.session.pendingBox; + final List subjects = tracking ? const [] : t.subjects; final Rect? drawing = _from != null && _to != null ? Rect.fromPoints(_from!, _to!) : null; return GestureDetector( key: const ValueKey('track-layer'), behavior: HitTestBehavior.opaque, dragStartBehavior: DragStartBehavior.down, - onTapUp: tracking ? null : (TapUpDetails d) => widget.session.setPendingBox(VisionGeometry.tapBoxAt(d.localPosition, view)), + onTapUp: tracking ? null : (TapUpDetails d) => _tap(d.localPosition, view, subjects), onPanStart: tracking ? null : (DragStartDetails d) => setState(() { @@ -65,9 +92,15 @@ class _ActiveTrackLayerState extends State { }, child: Stack(children: [ const SizedBox.expand(), + for (final SensedSubject s in subjects) + if (s.index != widget.session.pendingSubject) + Positioned.fromRect( + rect: VisionGeometry.toScreen(s.rect, view), + child: _Box(key: ValueKey('subject-${s.index}'), color: Glass.ink, dashed: true), + ), if (drawing != null) Positioned.fromRect(rect: drawing, child: const _Box(color: Glass.ink, dashed: true)), - if (pending != null) ..._pending(VisionGeometry.toScreen(pending, view), view), - if (tracking && t.rect != null) ..._tracked(t, VisionGeometry.toScreen(t.rect!, view), view), + if (pending != null) ..._pending(VisionGeometry.toScreen(pending, view), full), + if (tracking && t.rect != null) ..._tracked(t, VisionGeometry.toScreen(t.rect!, view), full), ]), ); }, @@ -82,7 +115,7 @@ class _ActiveTrackLayerState extends State { key: const ValueKey('track-go'), label: 'GO', color: PV.accent, - onTap: () => widget.onGo(widget.session.pendingBox!), + onTap: () => widget.onGo(widget.session.pendingBox!, widget.session.pendingSubject), ), const SizedBox(width: 6), _RoundButton( @@ -152,26 +185,33 @@ Widget _beside(Rect r, Size view, Widget child) { /// fly, then GO. While flying, the aircraft's own heading point is drawn and /// a new tap re-aims it. class TapFlyLayer extends StatelessWidget { - const TapFlyLayer({super.key, required this.mission, required this.session, required this.onGo}); + const TapFlyLayer({super.key, required this.mission, required this.session, required this.video, required this.onGo}); final MissionState mission; final VisionSession session; + + /// The feed's picture size, for the letterbox. + final ValueListenable video; final ValueChanged onGo; @override Widget build(BuildContext context) { return LayoutBuilder(builder: (BuildContext context, BoxConstraints c) { - final Size view = c.biggest; + final Size full = c.biggest; return ListenableBuilder( - listenable: Listenable.merge([mission, session]), + listenable: Listenable.merge([mission, session, video]), builder: (BuildContext context, _) { + final Rect view = VisionGeometry.contentRect(full, video.value); final TapFlyProgress f = mission.tapFly; final Offset? pending = session.pendingPoint; final Offset? heading = f.active ? f.image : null; return GestureDetector( key: const ValueKey('tapfly-layer'), behavior: HitTestBehavior.opaque, - onTapUp: (TapUpDetails d) => session.setPendingPoint(VisionGeometry.normalise(d.localPosition, view)), + onTapUp: (TapUpDetails d) { + final Offset? p = VisionGeometry.normalise(d.localPosition, view); + if (p != null) session.setPendingPoint(p); // a tap on a bar is no target + }, child: Stack(children: [ const SizedBox.expand(), if (heading != null) _marker(VisionGeometry.pointToScreen(heading, view), HudColors.ready, key: 'tapfly-heading'), @@ -179,7 +219,7 @@ class TapFlyLayer extends StatelessWidget { _marker(VisionGeometry.pointToScreen(pending, view), PV.accent, key: 'tapfly-target'), _beside( Rect.fromCenter(center: VisionGeometry.pointToScreen(pending, view), width: 40, height: 40), - view, + full, Row(mainAxisSize: MainAxisSize.min, children: [ _RoundButton(key: const ValueKey('tapfly-go'), label: 'GO', color: PV.accent, onTap: () => onGo(pending)), const SizedBox(width: 6), diff --git a/Fly App/lib/ui/fly/modes/vision/vision_panel.dart b/Fly App/lib/ui/fly/modes/vision/vision_panel.dart index e58ab96..6593312 100644 --- a/Fly App/lib/ui/fly/modes/vision/vision_panel.dart +++ b/Fly App/lib/ui/fly/modes/vision/vision_panel.dart @@ -87,7 +87,13 @@ class VisionModePanel extends StatelessWidget { note = f.bypassLabel; } else { final ActiveTrackProgress t = ms.activeTrack; - status = t.active ? t.stateLabel.toUpperCase() : (session.pendingBox != null ? 'TAP GO' : 'DRAW A BOX ROUND THE SUBJECT'); + status = t.active + ? t.stateLabel.toUpperCase() + : session.pendingBox != null + ? 'TAP GO' + : t.autoSensing + ? (t.subjects.isEmpty ? 'LOOKING FOR SUBJECTS' : 'TAP A SUBJECT') + : 'DRAW A BOX ROUND THE SUBJECT'; statusColor = t.cannotConfirm ? HudColors.danger : (t.active ? (t.following ? HudColors.ready : HudColors.caution) : HudText.dim); note = t.cannotConfirm ? t.reasonLabel : null; } @@ -114,6 +120,28 @@ class VisionModePanel extends StatelessWidget { if (active) _guard('Switch ActiveTrack mode', () => gate.switchActiveTrackMode(m)); }, ), + // Trace only: fly backwards ahead of a subject coming towards the aircraft. + if (session.trackMode == TrackMode.trace && model.profile.hasObstacleSensing) + HudChip( + key: const ValueKey('track-retreat'), + label: session.retreat == true ? 'Backward on' : 'Backward off', + selected: session.retreat == true, + onTap: () { + final bool on = session.retreat != true; + _guard('ActiveTrack backward', () async { + await gate.setTrackRetreat(on); + session.setRetreat(on); + }); + }, + ), + // The aircraft finds subjects itself; tap one instead of drawing. + if (ms.activeTrack.autoSensingSupported && !active) + HudChip( + key: const ValueKey('track-autosense'), + label: 'Auto-detect', + selected: ms.activeTrack.autoSensing, + onTap: () => _guard('Auto-detect', () => gate.setAutoSensing(!ms.activeTrack.autoSensing)), + ), ]; return Container( @@ -156,6 +184,14 @@ class VisionModePanel extends StatelessWidget { if (active) _guard('TapFly speed', () => gate.setTapFlySpeed(v)); }, ), + // Free lets the pilot turn away; this turns it back to face the way it flies. + if (active && session.direction == TapFlyDirection.free) + HudChip( + key: const ValueKey('tapfly-reset-heading'), + label: 'Reset heading', + selected: false, + onTap: () => _guard('Reset heading', gate.resetTapFlyHeading), + ), ], ]), ]), diff --git a/Fly App/lib/ui/fly/overlays/camera_touch_layer.dart b/Fly App/lib/ui/fly/overlays/camera_touch_layer.dart index 8ea2c27..63cd94b 100644 --- a/Fly App/lib/ui/fly/overlays/camera_touch_layer.dart +++ b/Fly App/lib/ui/fly/overlays/camera_touch_layer.dart @@ -1,10 +1,12 @@ import 'dart:async'; +import 'package:flutter/foundation.dart' show ValueListenable; import 'package:flutter/material.dart'; import '../../../dji_service.dart'; import '../../../fly/fly_prefs.dart'; import '../../../fly/hud_features.dart'; +import '../../../fly/modes/vision_modes.dart' show VisionGeometry; import '../../../fly/product_profile.dart'; import '../../../fly/state/camera_state.dart'; import '../../../theme.dart'; @@ -19,17 +21,27 @@ import '../panels/camera_write.dart'; /// "Exposure" box with a × that returns metering to centre-weighted; the box /// stays while the camera reports SPOT metering. /// -/// Positions are sent as 0..1 across this layer, which covers the full-bleed -/// video; with a letterboxed feed the edges are slightly off, which the -/// camera's coarse metering grid absorbs. +/// Positions are sent as 0..1 of the picture: the native view letterboxes the +/// feed to [video]'s aspect ratio, so taps are mapped through +/// [VisionGeometry.contentRect], and a tap on a black bar does nothing. class CameraTouchLayer extends StatefulWidget { - const CameraTouchLayer({super.key, required this.camera, required this.dji, required this.connected, required this.onError}); + const CameraTouchLayer({ + super.key, + required this.camera, + required this.dji, + required this.connected, + required this.onError, + this.video, + }); final CameraState camera; final DjiService dji; final bool connected; final void Function(String label, Object error) onError; + /// The feed's picture size; null fills the layer. + final ValueListenable? video; + @override State createState() => _CameraTouchLayerState(); } @@ -48,7 +60,9 @@ class _CameraTouchLayerState extends State { } Future _act(_Mark mark, Offset local, Size size) async { - final double x = (local.dx / size.width).clamp(0.0, 1.0), y = (local.dy / size.height).clamp(0.0, 1.0); + final Offset? at = VisionGeometry.normalise(local, VisionGeometry.contentRect(size, widget.video?.value)); + if (at == null) return; // on a letterbox bar + final double x = at.dx, y = at.dy; setState(() { if (mark == _Mark.focus) { _focusAt = local; diff --git a/Fly App/test/fly/fake_dji.dart b/Fly App/test/fly/fake_dji.dart index 0a647db..a55fc42 100644 --- a/Fly App/test/fly/fake_dji.dart +++ b/Fly App/test/fly/fake_dji.dart @@ -69,6 +69,16 @@ class FakeDjiService extends DjiService { @override Future setTapFlySpeed(double speed) => _record('setTapFlySpeed(${speed.toStringAsFixed(0)})'); @override + Future resetTapFlyHeading() => _record('resetTapFlyHeading'); + @override + Future setActiveTrackAutoSensing(bool on) => _record('setActiveTrackAutoSensing($on)'); + @override + Future startActiveTrackSubject(int index, {String mode = 'TRACE'}) => _record('startActiveTrackSubject($index, $mode)'); + @override + Future setActiveTrackRetreat(bool on) => _record('setActiveTrackRetreat($on)'); + @override + Future getActiveTrackRetreat() async => false; + @override Future stopActiveTrack() => _record('stopActiveTrack'); @override Future startCompassCalibration() => _record('startCompassCalibration'); diff --git a/Fly App/test/fly/vision_modes_test.dart b/Fly App/test/fly/vision_modes_test.dart index 31d2117..5f9477a 100644 --- a/Fly App/test/fly/vision_modes_test.dart +++ b/Fly App/test/fly/vision_modes_test.dart @@ -51,24 +51,50 @@ Map grounded() => telemetryTick() void main() { group('VisionGeometry', () { - const Size view = Size(800, 400); + const Rect view = Rect.fromLTWH(0, 0, 800, 400); - test('a drag becomes a 0..1 box whichever way it was drawn, clipped to the view', () { + test('the picture is aspect-fitted and centred; unknown size fills the view', () { + // 16:9 feed on a 19.5:9 phone: bars left and right. + final Rect wide = VisionGeometry.contentRect(const Size(844, 390), const Size(1920, 1080)); + expect(wide.height, 390); + expect(wide.width, closeTo(693.33, 0.01)); + expect(wide.left, closeTo((844 - 693.33) / 2, 0.01)); + // 4:3 feed on a 16:10 tablet: bars left and right too. + expect(VisionGeometry.contentRect(const Size(1280, 800), const Size(1440, 1080)).width, closeTo(1066.67, 0.01)); + // A feed wider than the view: bars top and bottom. + final Rect tall = VisionGeometry.contentRect(const Size(400, 400), const Size(1920, 1080)); + expect(tall.width, 400); + expect(tall.top, closeTo(87.5, 0.01)); + expect(VisionGeometry.contentRect(const Size(844, 390), null), const Rect.fromLTWH(0, 0, 844, 390)); + }); + + test('a drag becomes a 0..1 box whichever way it was drawn, clipped to the picture', () { final Rect r = VisionGeometry.dragBox(const Offset(600, 300), const Offset(200, -50), view)!; expect(r, const Rect.fromLTRB(0.25, 0, 0.75, 0.75)); }); - test('a small drag is no box', () { + test('a small drag, or one on a bar, is no box', () { expect(VisionGeometry.dragBox(const Offset(100, 100), const Offset(110, 200), view), isNull); + const Rect content = Rect.fromLTWH(100, 0, 600, 400); + expect(VisionGeometry.dragBox(const Offset(0, 50), const Offset(90, 200), content), isNull); }); - test('a tap makes a person-sized box, kept inside the view', () { - final Rect mid = VisionGeometry.tapBoxAt(const Offset(400, 200), view); + test('taps are relative to the picture, and nothing on a bar', () { + const Rect content = Rect.fromLTWH(100, 0, 600, 400); + expect(VisionGeometry.normalise(const Offset(400, 200), content), const Offset(0.5, 0.5)); + expect(VisionGeometry.normalise(const Offset(50, 200), content), isNull); + expect(VisionGeometry.tapBoxAt(const Offset(750, 200), content), isNull); + expect(VisionGeometry.pointToScreen(const Offset(0.5, 0.5), content), const Offset(400, 200)); + expect(VisionGeometry.toScreen(const Rect.fromLTRB(0, 0, 1, 1), content), content); + }); + + test('a tap makes a person-sized box, kept inside the picture', () { + final Rect mid = VisionGeometry.tapBoxAt(const Offset(400, 200), view)!; expect(mid.center, const Offset(0.5, 0.5)); expect(mid.width, closeTo(VisionGeometry.tapBox.width, 1e-9)); expect(mid.height, closeTo(VisionGeometry.tapBox.height, 1e-9)); - final Rect corner = VisionGeometry.tapBoxAt(Offset.zero, view); - expect(corner.left, 0); + final Rect corner = VisionGeometry.tapBoxAt(const Offset(1, 1), view)!; + expect(corner.left, closeTo(0, 1e-9)); expect(corner.top, closeTo(0, 1e-9)); }); }); @@ -203,6 +229,61 @@ void main() { }); }); + group('auto-sensing, retreat, reset heading, palm, video size', () { + test('sensed subjects arrive while auto-sensing, the flags stick across events', () { + final MissionState s = MissionState() + ..apply({'type': 'mission', 'operator': 'activeTrack', 'state': 'IDLE', 'autoSensingSupported': true, 'autoSensing': false}); + expect(s.activeTrack.autoSensingSupported, isTrue); + s.apply(trackEvent('AUTO_SENSING', tracking: { + 'subjects': [ + {'index': 3, 'type': 'HUMAN', 'rect': box}, + {'index': 4, 'type': 'CAR', 'rect': {'l': 0.1, 't': 0.1, 'r': 0.05, 'b': 0.2}}, // bad rect + ], + })..['autoSensing'] = true); + expect(s.activeTrack.subjects.single.index, 3); + expect(s.activeTrack.autoSensing, isTrue); + expect(s.activeTrack.autoSensingSupported, isTrue, reason: 'not repeated, still known'); + expect(s.activeTrack.stateLabel, 'Tap a subject'); + s.apply(trackEvent('AIRCRAFT_FOLLOWING', tracking: {'rect': box})); + expect(s.activeTrack.subjects, isEmpty); + }); + + test('auto-sensing needs support and no tracking; a subject starts by index', () async { + final FakeDjiService dji = FakeDjiService(); + final FlightModel m = modelWith(telemetryTick()); + final SafetyGate g = SafetyGate(model: m, dji: dji); + await expectLater(g.setAutoSensing(true), throwsA(isA())); + m.mission.apply({'type': 'mission', 'operator': 'activeTrack', 'state': 'IDLE', 'autoSensingSupported': true}); + await g.setAutoSensing(true); + await g.startActiveTrackSubject(3, TrackMode.profile); + expect(dji.calls, ['setActiveTrackAutoSensing(true)', 'startActiveTrackSubject(3, PROFILE)']); + }); + + test('backward needs obstacle sensing; reset heading needs TapFly flying', () async { + final FakeDjiService dji = FakeDjiService(); + final FlightModel noSensors = FlightModel()..connected = true; + TelemetryReducer(noSensors).apply(telemetryTick()); + await expectLater(SafetyGate(model: noSensors, dji: dji).setTrackRetreat(true), throwsA(isA())); + final FlightModel m = modelWith(telemetryTick()); + final SafetyGate g = SafetyGate(model: m, dji: dji); + await g.setTrackRetreat(true); + await expectLater(g.resetTapFlyHeading(), throwsA(isA())); + m.mission.apply(tapFlyEvent('EXECUTING')); + await g.resetTapFlyHeading(); + expect(dji.calls, ['setActiveTrackRetreat(true)', 'resetTapFlyHeading']); + }); + + test('the reducer takes the video size and the palm-control state', () { + final FlightModel m = modelWith(telemetryTick()); + TelemetryReducer(m) + ..apply({'type': 'videoSize', 'width': 1440, 'height': 1080}) + ..apply(telemetryTick()..['palmControl'] = 'FOLLOWING'); + expect(m.videoSize.value, const Size(1440, 1080)); + expect(m.vision.palmActive, isTrue); + expect(m.vision.palmLabel, 'Following your palm'); + }); + }); + group('FlyScreen (vision modes)', () { setUp(() { SharedPreferences.setMockInitialValues({}); @@ -263,6 +344,60 @@ void main() { expect(find.byKey(const ValueKey('track-panel')), findsNothing); }); + testWidgets('Auto-detect: tap a sensed subject, GO follows it by index', (WidgetTester tester) async { + final FlightModel m = modelWith(telemetryTick()); + m.mission.apply({'type': 'mission', 'operator': 'activeTrack', 'state': 'IDLE', 'autoSensingSupported': true}); + final FakeDjiService dji = await pump(tester, m); + await open(tester, 'activeTrack'); + await tester.tap(find.byKey(const ValueKey('track-autosense'))); + await tester.pump(); + expect(dji.calls.last, 'setActiveTrackAutoSensing(true)'); + + TelemetryReducer(m).apply(trackEvent('AUTO_SENSING', tracking: { + 'subjects': [{'index': 7, 'type': 'HUMAN', 'rect': box}], + })..['autoSensing'] = true); + await tester.pump(); + expect(find.byKey(const ValueKey('subject-7')), findsOneWidget); + // The box only draws; the layer underneath takes the tap. + await tester.tapAt(tester.getCenter(find.byKey(const ValueKey('subject-7')))); + await tester.pump(); + await tester.tap(find.byKey(const ValueKey('track-go'))); + await tester.pumpAndSettle(); + expect(dji.calls.last, 'startActiveTrackSubject(7, TRACE)'); + }); + + testWidgets('Backward toggles retreat for Trace', (WidgetTester tester) async { + final FlightModel m = modelWith(telemetryTick()); + final FakeDjiService dji = await pump(tester, m); + await open(tester, 'activeTrack'); + await tester.tap(find.byKey(const ValueKey('track-retreat'))); + await tester.pump(); + expect(dji.calls.last, 'setActiveTrackRetreat(true)'); + expect(find.text('Backward on'), findsOneWidget); + }); + + testWidgets('with a 4:3 feed, a tap on the black bar is no TapFly target', (WidgetTester tester) async { + final FlightModel m = modelWith(telemetryTick()); + m.videoSize.value = const Size(1440, 1080); // 520 px wide on a 390 px tall screen: bars of 162 px + await pump(tester, m); + await open(tester, 'tapFly'); + await tester.tapAt(const Offset(780, 200)); + await tester.pump(); + expect(find.byKey(const ValueKey('tapfly-target')), findsNothing); + await tester.tapAt(const Offset(500, 200)); + await tester.pump(); + expect(find.byKey(const ValueKey('tapfly-target')), findsOneWidget); + }); + + testWidgets('palm control shows while the aircraft follows a palm', (WidgetTester tester) async { + final FlightModel m = modelWith(telemetryTick()); + await pump(tester, m); + expect(find.byKey(const ValueKey('palm-control')), findsNothing); + TelemetryReducer(m).apply(telemetryTick()..['palmControl'] = 'FOLLOWING'); + await tester.pump(); + expect(find.byKey(const ValueKey('palm-control')), findsOneWidget); + }); + testWidgets('a vision mode that would warn asks for the slide first', (WidgetTester tester) async { final FlightModel m = modelWith(telemetryTick()); final FakeDjiService dji = await pump(tester, m);