Body tracking

Description

In this example we will look at how ARKit can detect a body in the scene and track its major joints as it moves around. We will also see how we can superimpose nodes onto the locations of those detected joints in 3D space.

For this we will use ARBodyTrackingConfiguration and we will keep track of the joints we detect in a Dictionary, placing spheres in their location. When the position of those joints change, we detect this and update the position of the spheres.

I was unable to find an example of Body Tracking using C#, Xamarin and ARKit anywhere online. I hope you find this helpful.

Video

Code

using ARKit;
using Foundation;
using OpenTK;
using SceneKit;
using System;
using System.Collections.Generic;
using UIKit;

namespace XamarinArkitSample
{
    public partial class BodyDetectionViewController : UIViewController
    {
        private readonly ARSCNView sceneView;

        public BodyDetectionViewController()
        {
            this.sceneView = new ARSCNView
            {
                AutoenablesDefaultLighting = true,
                Delegate = new SceneViewDelegate()
            };

            this.View.AddSubview(this.sceneView);
        }

        public override void ViewDidLoad()
        {
            base.ViewDidLoad();

            this.sceneView.Frame = this.View.Frame;
        }

        public override void ViewDidAppear(bool animated)
        {
            base.ViewDidAppear(animated);

            var bodyTrackingConfiguration = new ARBodyTrackingConfiguration()
            {
                WorldAlignment = ARWorldAlignment.Gravity
            };

            this.sceneView.Session.Run(bodyTrackingConfiguration);
        }

        public override void ViewDidDisappear(bool animated)
        {
            base.ViewDidDisappear(animated);
            this.sceneView.Session.Pause();
        }

        public override void DidReceiveMemoryWarning()
        {
            base.DidReceiveMemoryWarning();
        }

        public class SceneViewDelegate : ARSCNViewDelegate
        { 
            Dictionary<string, JointNode> joints = new Dictionary<string, JointNode>();

            public override void DidAddNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
            {
                if (!(anchor is ARBodyAnchor bodyAnchor))
                    return;

                foreach (var jointName in ARSkeletonDefinition.DefaultBody3DSkeletonDefinition.JointNames)
                {
                    JointNode jointNode = MakeJoint(jointName);

                    var jointPosition = GetJointPosition(bodyAnchor, jointName);
                    jointNode.Position = jointPosition;
                           
                    if (!joints.ContainsKey(jointName))
                    {
                        node.AddChildNode(jointNode);
                        joints.Add(jointName, jointNode);
                    }
                }
            }

            public override void DidUpdateNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
            {
                if (!(anchor is ARBodyAnchor bodyAnchor))
                    return;

                foreach (var jointName in ARSkeletonDefinition.DefaultBody3DSkeletonDefinition.JointNames)
                {       
                    var jointPosition = GetJointPosition(bodyAnchor, jointName);

                    if (joints.ContainsKey(jointName))
                    {
                        joints[jointName].Update(jointPosition);
                    }   
                }
            }

            private SCNVector3 GetJointPosition(ARBodyAnchor bodyAnchor, string jointName)
            {
                NMatrix4 jointTransform = bodyAnchor.Skeleton.GetModelTransform((NSString)jointName);
                return new SCNVector3(jointTransform.Column3);
            }

            private JointNode MakeJoint(string jointName)
            {
                var jointNode = new JointNode();

                var material = new SCNMaterial();
                material.Diffuse.Contents = GetJointColour(jointName);

                var jointGeometry = SCNSphere.Create(GetJointRadius(jointName));
                jointGeometry.FirstMaterial = material;
                jointNode.Geometry = jointGeometry;

                return jointNode;
            }

            private UIColor GetJointColour(string jointName)
            {
                switch (jointName)
                {
                    case "root":
                    case "left_foot_joint":
                    case "right_foot_joint":
                    case "left_leg_joint":
                    case "right_leg_joint":
                    case "left_hand_joint":
                    case "right_hand_joint":
                    case "left_arm_joint":
                    case "right_arm_joint":
                    case "left_forearm_joint":
                    case "right_forearm_joint":
                    case "head_joint":
                        return UIColor.Green;
                }

                return UIColor.White;
            }

            private float GetJointRadius(string jointName)
            {
                switch (jointName)
                {
                    case "root":
                    case "left_foot_joint":
                    case "right_foot_joint":
                    case "left_leg_joint":
                    case "right_leg_joint":
                    case "left_hand_joint":
                    case "right_hand_joint":
                    case "left_arm_joint":
                    case "right_arm_joint":
                    case "left_forearm_joint":
                    case "right_forearm_joint":
                    case "head_joint":
                        return 0.04f;
                }

                if (jointName.Contains("hand"))
                    return 0.01f;

                return 0.02f;
            }
        }

        public class JointNode : SCNNode
        {
            public void Update(SCNVector3 position)
            {
                this.Position = position;
            }
        }
    }

    public static class Extensions
    {
        public static SCNMatrix4 ToSCNMatrix4(this NMatrix4 self)
        {
            var row0 = new SCNVector4(self.M11, self.M12, self.M13, self.M14);
            var row1 = new SCNVector4(self.M21, self.M22, self.M23, self.M24);
            var row2 = new SCNVector4(self.M31, self.M32, self.M33, self.M34);
            var row3 = new SCNVector4(self.M41, self.M42, self.M43, self.M44);
            return new SCNMatrix4(row0, row1, row2, row3);
        }
    }
}