From c13cc7cc5c8ac8d972d6e0eaf42bb238f814386b Mon Sep 17 00:00:00 2001 From: Kenneth Yang Date: Mon, 29 Dec 2025 13:41:42 -0800 Subject: [PATCH 01/12] WIP drew trajectory --- .../AutomationInspectorViewModel.cs | 534 ++++++++++++++++-- 1 file changed, 499 insertions(+), 35 deletions(-) diff --git a/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs b/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs index 7f5cbf09..d3e95a1a 100644 --- a/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs +++ b/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs @@ -1,9 +1,14 @@ +using System; using System.Collections.Generic; using System.ComponentModel; +using System.Globalization; using System.Linq; using BrainAtlas; +using EphysLink; +using KS.Diagnostics; using Models; using Models.Scene; +using Pinpoint.Probes; using Services; using Unity.AppUI.MVVM; using Unity.AppUI.Redux; @@ -85,6 +90,50 @@ public partial class AutomationInspectorViewModel #endregion + #region Trajectory and Visualization Fields + + /// + /// Trajectory broken into 3 stages (for 3 axes of movement). + /// + /// Execution order: DV, AP, ML. Defaults to negative infinity when there is no trajectory. + private (Vector3 first, Vector3 second, Vector3 third) _trajectoryCoordinates = ( + Vector3.negativeInfinity, + Vector3.negativeInfinity, + Vector3.negativeInfinity + ); + + /// + /// Record the depth at the entry coordinate. + /// + /// Used during insertion to calculate the actual distance needed to retract back to the entry coordinate. + private float _entryCoordinateDepth; + + /// + /// Trajectory line GameObjects. + /// + private (GameObject ap, GameObject ml, GameObject dv) _trajectoryLineGameObjects; + + /// + /// Trajectory line renderers. + /// + private (LineRenderer ap, LineRenderer ml, LineRenderer dv) _trajectoryLineRenderers; + + // Axes colors + private static readonly Color AP_COLOR = new(1, 0.3215686f, 0.3215686f); // Red + private static readonly Color ML_COLOR = new(0.2039216f, 0.6745098f, 0.8784314f); // Blue + private static readonly Color DV_COLOR = new(1, 0.854902f, 0.4745098f); // Yellow + + // Trajectory line properties + private const float LINE_WIDTH = 0.1f; + private const int NUM_SEGMENTS = 2; + + // Safety margin + private const float IDEAL_ENTRY_COORDINATE_TO_DURA_DISTANCE = 3.5f; + + // Movement speed + private const float AUTOMATIC_MOVEMENT_SPEED = 0.5f; // mm/s + #endregion + public AutomationInspectorViewModel( StoreService storeService, EphysLinkService ephysLinkService @@ -161,10 +210,13 @@ private void OnSceneStateChanged(SceneState state) var selectedTargetInsertionProbeState = state.Probes.FirstOrDefault(probeState => probeState.Name == state.ActiveManipulatorState.TargetInsertionProbeName ); - TargetInsertionProbeIndex = (selectedTargetInsertionProbeState == null - || !TargetInsertionProbeStates.Contains(selectedTargetInsertionProbeState)) - ? -1 - : TargetInsertionProbeStates.IndexOf(selectedTargetInsertionProbeState); + TargetInsertionProbeIndex = + ( + selectedTargetInsertionProbeState == null + || !TargetInsertionProbeStates.Contains(selectedTargetInsertionProbeState) + ) + ? -1 + : TargetInsertionProbeStates.IndexOf(selectedTargetInsertionProbeState); // Update dura offset. DuraOffset = state.ActiveManipulatorState.DuraOffset; @@ -202,6 +254,9 @@ private void OnShuttingDown() { App.shuttingDown -= OnShuttingDown; _sceneStateSubscription.Dispose(); + + // Clean up trajectory lines + RemoveTrajectoryLines(); } #region Commands @@ -232,17 +287,36 @@ private void SelectTargetInsertionProbe(int index) ResetTargetInsertionProbeSelection(); return; } - - // Set the selected target insertion probe name in the store. - var selectedTargetInsertionProbeState = TargetInsertionProbeStates.ElementAt( - index + + // Get the selected target insertion probe + var selectedTargetInsertionProbeState = TargetInsertionProbeStates.ElementAt(index); + + // Get the probe manager for the selected target + var targetProbeManager = ProbeManager.Instances.FirstOrDefault(manager => + manager.name == selectedTargetInsertionProbeState.Name ); + + if (targetProbeManager == null) + { + Debug.LogError( + $"Target probe manager not found: {selectedTargetInsertionProbeState.Name}" + ); + return; + } + + // Set the selected target insertion probe name in the store _storeService.Store.Dispatch( SceneActions.SET_TARGET_INSERTION_PROBE_NAME, (ActiveManipulatorId, selectedTargetInsertionProbeState.Name) ); - // TODO: Call ComputeEntryCoordinateTrajectory once it has been converted. + // Compute and visualize trajectory with the target probe manager + var entryCoordinate = ComputeTargetEntryCoordinateTrajectory(targetProbeManager); + + if (float.IsNegativeInfinity(entryCoordinate.x)) + { + Debug.LogWarning("Failed to compute trajectory for selected target probe"); + } } [ICommand] @@ -253,44 +327,189 @@ private void ResetTargetInsertionProbeSelection() SceneActions.SET_TARGET_INSERTION_PROBE_NAME, (ActiveManipulatorId, string.Empty) ); + + // Remove trajectory visualization + RemoveTrajectoryLines(); } [ICommand] - private void DriveToTargetEntryCoordinate() + private async void DriveToTargetEntryCoordinate() { - ProbeService - .DriveActiveProbeToTargetEntryCoordinate() - .ContinueWith(task => + // Validate trajectory exists + if (float.IsNegativeInfinity(_trajectoryCoordinates.first.x)) + { + Debug.LogError($"No trajectory planned for manipulator {ActiveManipulatorId}"); + return; + } + + // Convert coordinates to manipulator positions + var dvPosition = ConvertInsertionAPMLDVToManipulatorPosition( + _trajectoryCoordinates.first + ); + var apPosition = ConvertInsertionAPMLDVToManipulatorPosition( + _trajectoryCoordinates.second + ); + var mlPosition = ConvertInsertionAPMLDVToManipulatorPosition( + _trajectoryCoordinates.third + ); + + // Check if conversion failed + if (dvPosition == null || apPosition == null || mlPosition == null) + { + HandleDriveFailed( + "Failed to convert trajectory coordinates to manipulator positions" + ); + return; + } + + // Set state to driving + _storeService.Store.Dispatch( + SceneActions.SET_AUTOMATION_PROGRESS_STATE, + (ActiveManipulatorId, AutomationProgressState.DrivingToTargetEntryCoordinate) + ); + + // Log that movement is starting + OutputLog.Log( + new[] { - // Do not proceed if the drive failed. - if (!task.Result) - return; + "Automation", + DateTime.Now.ToString(CultureInfo.InvariantCulture), + "DriveToTargetEntryCoordinate", + ActiveManipulatorId, + "Start", + } + ); - // Complete the drive state if successful. - _storeService.Store.Dispatch( - SceneActions.COMPLETE_AUTOMATION_INTERMEDIATE_PROGRESS, - ActiveManipulatorId - ); - }); + // Stage 1: DV movement + var dvResponse = await _ephysLinkService.SetPosition( + new SetPositionRequest( + ActiveManipulatorId, + dvPosition.Value, + AUTOMATIC_MOVEMENT_SPEED + ) + ); + + if (EphysLinkService.HasError(dvResponse.Error)) + { + HandleDriveFailed("Failed to move to DV position"); + return; + } + + // Stage 2: AP movement + var apResponse = await _ephysLinkService.SetPosition( + new SetPositionRequest( + ActiveManipulatorId, + apPosition.Value, + AUTOMATIC_MOVEMENT_SPEED + ) + ); + + if (EphysLinkService.HasError(apResponse.Error)) + { + HandleDriveFailed("Failed to move to AP position"); + return; + } + + // Stage 3: ML movement + var mlResponse = await _ephysLinkService.SetPosition( + new SetPositionRequest( + ActiveManipulatorId, + mlPosition.Value, + AUTOMATIC_MOVEMENT_SPEED + ) + ); + + if (EphysLinkService.HasError(mlResponse.Error)) + { + HandleDriveFailed("Failed to move to ML position"); + return; + } + + // Record entry coordinate depth + var finalPositionResponse = await _ephysLinkService.GetPosition(ActiveManipulatorId); + if (EphysLinkService.HasError(finalPositionResponse.Error)) + { + HandleDriveFailed("Failed to get final position at entry coordinate"); + return; + } + + _entryCoordinateDepth = finalPositionResponse.Position.w; + + // Remove visualization lines + RemoveTrajectoryLines(); + + // Complete the drive state + _storeService.Store.Dispatch( + SceneActions.COMPLETE_AUTOMATION_INTERMEDIATE_PROGRESS, + ActiveManipulatorId + ); + + // Log completion + OutputLog.Log( + new[] + { + "Automation", + DateTime.Now.ToString(CultureInfo.InvariantCulture), + "DriveToTargetEntryCoordinate", + ActiveManipulatorId, + "Finish", + } + ); + } + + private void HandleDriveFailed(string errorMessage) + { + Debug.LogError(errorMessage); + OutputLog.Log( + new[] + { + "Automation", + DateTime.Now.ToString(CultureInfo.InvariantCulture), + "DriveToTargetEntryCoordinate", + ActiveManipulatorId, + $"Failed: {errorMessage}", + } + ); + + // Revert state to calibrated + _storeService.Store.Dispatch( + SceneActions.CANCEL_AUTOMATION_INTERMEDIATE_PROGRESS, + ActiveManipulatorId + ); } [ICommand] - private void StopDriveToTargetEntryCoordinate() + private async void StopDriveToTargetEntryCoordinate() { - ProbeService - .StopActiveProbeDriveToTargetEntryCoordinate() - .ContinueWith(task => + // Log stop request + OutputLog.Log( + new[] { - // Do not proceed if the drive failed. - if (!task.Result) - return; + "Automation", + DateTime.Now.ToString(CultureInfo.InvariantCulture), + "DriveToTargetEntryCoordinate", + ActiveManipulatorId, + "Stopped", + } + ); - // Reset back to calibrated state. - _storeService.Store.Dispatch( - SceneActions.SET_AUTOMATION_PROGRESS_STATE, - (ActiveManipulatorId, AutomationProgressState.IsCalibrated) - ); - }); + // Send stop command to manipulator + var stopResponse = await _ephysLinkService.Stop(ActiveManipulatorId); + + // Check for errors + if (EphysLinkService.HasError(stopResponse)) + { + Debug.LogError($"Failed to stop drive: {stopResponse}"); + return; + } + + // Cancel intermediate progress (revert to IsCalibrated state) + _storeService.Store.Dispatch( + SceneActions.CANCEL_AUTOMATION_INTERMEDIATE_PROGRESS, + ActiveManipulatorId + ); + + // Note: Do NOT remove trajectory lines on stop - user may want to restart } [ICommand] @@ -372,5 +591,250 @@ private void StopInsertionDrive() } #endregion + + #region Trajectory Computation and Visualization + + /// + /// Compute the entry coordinate and trajectory for the target insertion. Also, create and update the trajectory visualization lines. + /// + /// The probe manager of the target insertion probe. + /// + /// The computed entry coordinate in AP, ML, DV coordinates. Vector3.negativeInfinity if target is unset or computation fails. + /// + private Vector3 ComputeTargetEntryCoordinateTrajectory(ProbeManager targetProbeManager) + { + // Validate target probe manager + if (targetProbeManager == null) + { + Debug.LogError("Target probe manager is null"); + RemoveTrajectoryLines(); + return Vector3.negativeInfinity; + } + + // Check if BrainAtlasManager is ready + if ( + BrainAtlasManager.Instance == null + || BrainAtlasManager.ActiveReferenceAtlas == null + ) + { + Debug.LogWarning("BrainAtlasManager not ready for trajectory computation"); + RemoveTrajectoryLines(); + return Vector3.negativeInfinity; + } + + // Compute entry coordinate in world space + var surfaceCoordinateWorldT = targetProbeManager.GetSurfaceCoordinateWorldT(); + var tipForwardWorldU = targetProbeManager + .ProbeController.GetTipWorldU() + .tipForwardWorldU; + + var entryCoordinateWorld = + surfaceCoordinateWorldT + - tipForwardWorldU * IDEAL_ENTRY_COORDINATE_TO_DURA_DISTANCE; + + // Convert to AP/ML/DV coordinates + var entryCoordinateAtlasU = BrainAtlasManager.ActiveReferenceAtlas.World2Atlas( + entryCoordinateWorld + ); + var entryCoordinateAPMLDV = BrainAtlasManager.ActiveAtlasTransform.U2T( + entryCoordinateAtlasU + ); + + // Get current manipulator coordinate. + var currentState = _storeService.Store.GetState(SliceNames.SCENE_SLICE); + var visualizationProbeName = currentState.ActiveManipulatorState.VisualizationProbeName; + var currentCoordinate = currentState + .Probes.FirstOrDefault(state => state.Name == visualizationProbeName)! + .APMLDV; + + // Create 3-stage trajectory + // Stage 1: DV movement (change depth only) + _trajectoryCoordinates.first = new Vector3( + currentCoordinate.x, + currentCoordinate.y, + entryCoordinateAPMLDV.z + ); + + // Stage 2: AP movement (change AP only) + _trajectoryCoordinates.second = new Vector3( + entryCoordinateAPMLDV.x, + currentCoordinate.y, + entryCoordinateAPMLDV.z + ); + + // Stage 3: ML movement (final position) + _trajectoryCoordinates.third = entryCoordinateAPMLDV; + + // Create and update trajectory lines + CreateTrajectoryLines(); + UpdateTrajectoryLines(); + + // Return final entry coordinate + return _trajectoryCoordinates.third; + } + + /// + /// Create the trajectory line game objects and line renderers (if needed). + /// + private void CreateTrajectoryLines() + { + // Exit if they already exist + if (_trajectoryLineGameObjects.ap != null) + return; + + // Create the trajectory line game objects + _trajectoryLineGameObjects = ( + new GameObject("APTrajectoryLine") { layer = 5 }, + new GameObject("MLTrajectoryLine") { layer = 5 }, + new GameObject("DVTrajectoryLine") { layer = 5 } + ); + + // Create the line renderers + _trajectoryLineRenderers = ( + _trajectoryLineGameObjects.ap.AddComponent(), + _trajectoryLineGameObjects.ml.AddComponent(), + _trajectoryLineGameObjects.dv.AddComponent() + ); + + // Apply materials + var defaultSpriteShader = Shader.Find("Sprites/Default"); + _trajectoryLineRenderers.ap.material = new Material(defaultSpriteShader) + { + color = AP_COLOR, + }; + _trajectoryLineRenderers.ml.material = new Material(defaultSpriteShader) + { + color = ML_COLOR, + }; + _trajectoryLineRenderers.dv.material = new Material(defaultSpriteShader) + { + color = DV_COLOR, + }; + + // Set line widths + _trajectoryLineRenderers.ap.startWidth = _trajectoryLineRenderers.ap.endWidth = + LINE_WIDTH; + _trajectoryLineRenderers.ml.startWidth = _trajectoryLineRenderers.ml.endWidth = + LINE_WIDTH; + _trajectoryLineRenderers.dv.startWidth = _trajectoryLineRenderers.dv.endWidth = + LINE_WIDTH; + + // Set segment counts + _trajectoryLineRenderers.ap.positionCount = NUM_SEGMENTS; + _trajectoryLineRenderers.ml.positionCount = NUM_SEGMENTS; + _trajectoryLineRenderers.dv.positionCount = NUM_SEGMENTS; + } + + /// + /// Update the trajectory line positions. + /// + private void UpdateTrajectoryLines() + { + if ( + BrainAtlasManager.Instance == null + || BrainAtlasManager.ActiveReferenceAtlas == null + ) + return; + + // Get active probe manager for current tip position + var sceneState = _storeService.Store.GetState(SliceNames.SCENE_SLICE); + var activeProbeManager = ProbeManager.Instances.FirstOrDefault(manager => + manager.name == sceneState.ActiveManipulatorState.VisualizationProbeName + ); + + if (activeProbeManager == null) + return; + + // DV line: From current probe tip to first coordinate + _trajectoryLineRenderers.dv.SetPosition( + 0, + activeProbeManager.ProbeController.ProbeTipT.position + ); + _trajectoryLineRenderers.dv.SetPosition( + 1, + BrainAtlasManager.ActiveReferenceAtlas.Atlas2World( + BrainAtlasManager.ActiveAtlasTransform.T2U_Vector(_trajectoryCoordinates.first) + ) + ); + + // AP line: From first to second coordinate + _trajectoryLineRenderers.ap.SetPosition( + 0, + BrainAtlasManager.ActiveReferenceAtlas.Atlas2World( + BrainAtlasManager.ActiveAtlasTransform.T2U_Vector(_trajectoryCoordinates.first) + ) + ); + _trajectoryLineRenderers.ap.SetPosition( + 1, + BrainAtlasManager.ActiveReferenceAtlas.Atlas2World( + BrainAtlasManager.ActiveAtlasTransform.T2U_Vector(_trajectoryCoordinates.second) + ) + ); + + // ML line: From second to third coordinate + _trajectoryLineRenderers.ml.SetPosition( + 0, + BrainAtlasManager.ActiveReferenceAtlas.Atlas2World( + BrainAtlasManager.ActiveAtlasTransform.T2U_Vector(_trajectoryCoordinates.second) + ) + ); + _trajectoryLineRenderers.ml.SetPosition( + 1, + BrainAtlasManager.ActiveReferenceAtlas.Atlas2World( + BrainAtlasManager.ActiveAtlasTransform.T2U_Vector(_trajectoryCoordinates.third) + ) + ); + } + + /// + /// Destroy the trajectory line game objects and line renderers. Also reset the references. + /// + private void RemoveTrajectoryLines() + { + // Destroy the objects + if (_trajectoryLineGameObjects.ap != null) + UnityEngine.Object.Destroy(_trajectoryLineGameObjects.ap); + if (_trajectoryLineGameObjects.ml != null) + UnityEngine.Object.Destroy(_trajectoryLineGameObjects.ml); + if (_trajectoryLineGameObjects.dv != null) + UnityEngine.Object.Destroy(_trajectoryLineGameObjects.dv); + + // Reset the references + _trajectoryLineGameObjects = (null, null, null); + _trajectoryLineRenderers = (null, null, null); + + // Reset trajectory coordinates + _trajectoryCoordinates = ( + Vector3.negativeInfinity, + Vector3.negativeInfinity, + Vector3.negativeInfinity + ); + } + + /// + /// Convert insertion AP, ML, DV coordinates to manipulator translation stage position. + /// + /// AP, ML, DV coordinates from an insertion. + /// Computed manipulator translation stage positions to match this coordinate, or null if conversion fails. + private Vector4? ConvertInsertionAPMLDVToManipulatorPosition(Vector3 insertionAPMLDV) + { + var sceneState = _storeService.Store.GetState(SliceNames.SCENE_SLICE); + var activeProbeManager = ProbeManager.Instances.FirstOrDefault(manager => + manager.name == sceneState.ActiveManipulatorState.VisualizationProbeName + ); + + if (activeProbeManager == null || !activeProbeManager.IsEphysLinkControlled) + { + Debug.LogError("Active probe manager not found or not EphysLink controlled"); + return null; + } + + // Use the ManipulatorBehaviorController's conversion method + return activeProbeManager.ManipulatorBehaviorController.ConvertInsertionAPMLDVToManipulatorPosition( + insertionAPMLDV + ); + } + + #endregion } } From 49e7ef7b879d32b19d39427e24f43dadb39424fb Mon Sep 17 00:00:00 2001 From: Kenneth Yang Date: Mon, 29 Dec 2025 15:10:27 -0800 Subject: [PATCH 02/12] Drive to target insertion --- .../AutomationInspectorViewModel.cs | 55 ++++++++++++++++--- .../UI/Views/AutomationInspectorView.cs | 7 +++ 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs b/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs index d3e95a1a..fb14c61d 100644 --- a/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs +++ b/Assets/Scripts/UI/ViewModels/AutomationInspectorViewModel.cs @@ -4,10 +4,12 @@ using System.Globalization; using System.Linq; using BrainAtlas; +using BrainAtlas.CoordinateSystems; using EphysLink; using KS.Diagnostics; using Models; using Models.Scene; +using Pinpoint.CoordinateSystems; using Pinpoint.Probes; using Services; using Unity.AppUI.MVVM; @@ -819,20 +821,59 @@ private void RemoveTrajectoryLines() private Vector4? ConvertInsertionAPMLDVToManipulatorPosition(Vector3 insertionAPMLDV) { var sceneState = _storeService.Store.GetState(SliceNames.SCENE_SLICE); - var activeProbeManager = ProbeManager.Instances.FirstOrDefault(manager => - manager.name == sceneState.ActiveManipulatorState.VisualizationProbeName + var activeManipulatorState = sceneState.ActiveManipulatorState; + + // Validate we have the necessary data + if (BrainAtlasManager.ActiveReferenceAtlas == null) + { + Debug.LogError("BrainAtlasManager.ActiveReferenceAtlas is null"); + return null; + } + + if (sceneState.ManipulatorCoordinateSpace == null) + { + Debug.LogError("ManipulatorCoordinateSpace is null"); + return null; + } + + // Convert AP/ML/DV to world coordinate + var convertToWorld = BrainAtlasManager.ActiveReferenceAtlas.Atlas2World_Vector( + BrainAtlasManager.ActiveAtlasTransform.T2U_Vector(insertionAPMLDV) ); - if (activeProbeManager == null || !activeProbeManager.IsEphysLinkControlled) + // Create coordinate transform based on manipulator configuration + var numAxes = sceneState.NumberOfAxesOnManipulator; + var isRightHanded = activeManipulatorState.Handedness == ManipulatorHandedness.Right; + var yaw = activeManipulatorState.Angles.x; + var pitch = activeManipulatorState.Angles.y; + + CoordinateTransform coordinateTransform = numAxes switch + { + 4 => isRightHanded + ? new FourAxisRightHandedManipulatorTransform(yaw) + : new FourAxisLeftHandedManipulatorTransform(yaw), + 3 => new ThreeAxisLeftHandedTransform(yaw, pitch), + _ => null, + }; + + if (coordinateTransform == null) { - Debug.LogError("Active probe manager not found or not EphysLink controlled"); + Debug.LogError($"Unsupported number of axes: {numAxes}"); return null; } - // Use the ManipulatorBehaviorController's conversion method - return activeProbeManager.ManipulatorBehaviorController.ConvertInsertionAPMLDVToManipulatorPosition( - insertionAPMLDV + // Convert to manipulator space + var posInManipulatorSpace = sceneState.ManipulatorCoordinateSpace.World2Space( + convertToWorld ); + Vector4 posInManipulatorTransform = coordinateTransform.U2T(posInManipulatorSpace); + + // Apply brain surface offset (DuraOffset) + var duraOffset = activeManipulatorState.DuraOffset; + posInManipulatorTransform.w -= float.IsNaN(duraOffset) ? 0 : duraOffset; + + // Apply coordinate offsets and return result + return posInManipulatorTransform + activeManipulatorState.ReferenceCoordinateOffset; } #endregion diff --git a/Assets/Scripts/UI/Views/AutomationInspectorView.cs b/Assets/Scripts/UI/Views/AutomationInspectorView.cs index 9cca5824..c9fafd36 100644 --- a/Assets/Scripts/UI/Views/AutomationInspectorView.cs +++ b/Assets/Scripts/UI/Views/AutomationInspectorView.cs @@ -116,9 +116,16 @@ public AutomationInspectorView(AutomationInspectorViewModel automationInspectorV var targetEntryDriveButton = root.Q