Facial expression detection

Description

In this lesson, we look at the large number of facial expressions that ARKit face tracking can recognise.

Notice for the first time we are using material.FillMode = SCNFillMode.Lines which works well to show the contours of the face.

We interrogate a property on the detected FaceAnchor called BlendShapes to get information about different parts of the face.

While ARKit is able to track many more expressions than the code sample below, I have decided to detect the following expressions and change the colour of the FaceGeometry material lines accordingly.

• Smile - Pink
• Mouth Frown - Brown
• EyesWide - Green
• BrowUp - Cyan
• TongueOut - Red
• MouthFunnel - Yellow
• Eyes Left/Right - Magenta
• CheeksPuff - Orange
• EyesBlink - Blue

The full list of detectable available facial expressions is at ARFaceAnchor.BlendShapeLocation.

Video

Code

using ARKit;
using SceneKit;
using System;
using UIKit;

namespace XamarinArkitSample
{
    public partial class ViewController : UIViewController
    {
        private readonly ARSCNView sceneView;
        

        public ViewController(IntPtr handle) : base(handle)
        {
            this.sceneView = new ARSCNView
            {
                AutoenablesDefaultLighting = true,
                Delegate = new SceneViewDelegate()
            };

            this.View.AddSubview(this.sceneView);
        }

        public override void ViewDidLoad()
        {
            base.ViewDidLoad();

            this.sceneView.Frame = this.View.Frame;
        }

        public override void ViewDidAppear(bool animated)
        {
            base.ViewDidAppear(animated);

            var faceTrackingConfiguration = new ARFaceTrackingConfiguration()
            {
                LightEstimationEnabled = true,
                MaximumNumberOfTrackedFaces = 1
            };

            this.sceneView.Session.Run(faceTrackingConfiguration);
        }

        public override void ViewDidDisappear(bool animated)
        {
            base.ViewDidDisappear(animated);

            this.sceneView.Session.Pause();
        }

        public override void DidReceiveMemoryWarning()
        {
            base.DidReceiveMemoryWarning();
        }
    }

    public class SceneViewDelegate : ARSCNViewDelegate
    {
        public override void DidAddNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
        {
            if (anchor is ARFaceAnchor)
            {
                var faceGeometry = ARSCNFaceGeometry.Create(renderer.GetDevice());
                node.Geometry = faceGeometry;
                node.Geometry.FirstMaterial.FillMode = SCNFillMode.Lines;
            }
        }        

        public override void DidUpdateNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
        {
            if (anchor is ARFaceAnchor)
            {
                var faceAnchor = anchor as ARFaceAnchor;
                var faceGeometry = node.Geometry as ARSCNFaceGeometry;
                var expressionThreshold = 0.5f;

                faceGeometry.Update(faceAnchor.Geometry);

                if (faceAnchor.BlendShapes.EyeWideLeft > expressionThreshold
                    || faceAnchor.BlendShapes.EyeWideRight > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Green);
                    return;
                }

                if (faceAnchor.BlendShapes.EyeBlinkLeft > expressionThreshold
                    || faceAnchor.BlendShapes.EyeBlinkRight > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Blue);
                    return;
                }

                if (faceAnchor.BlendShapes.MouthFrownLeft > expressionThreshold
                    || faceAnchor.BlendShapes.MouthFrownRight > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Black);
                    return;
                }

                if (faceAnchor.BlendShapes.MouthSmileLeft > expressionThreshold
                    || faceAnchor.BlendShapes.MouthSmileRight > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.SystemPinkColor);
                    return;
                }

                if (faceAnchor.BlendShapes.BrowOuterUpLeft > expressionThreshold
                    || faceAnchor.BlendShapes.BrowOuterUpRight > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Cyan);
                    return;
                }

                if (faceAnchor.BlendShapes.EyeLookOutLeft > expressionThreshold
                    || faceAnchor.BlendShapes.EyeLookOutRight > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Magenta);
                    return;
                }

                if (faceAnchor.BlendShapes.TongueOut > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Red);
                    return;
                }

                if (faceAnchor.BlendShapes.MouthFunnel > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Yellow);
                    return;
                }

                if (faceAnchor.BlendShapes.CheekPuff > expressionThreshold)
                {
                    ChangeFaceColour(node, UIColor.Orange);
                    return;
                }

                ChangeFaceColour(node, UIColor.White);
            }
        }

        private void ChangeFaceColour(SCNNode faceGeometry, UIColor colour)
        {
            var material = new SCNMaterial();
            material.Diffuse.Contents = colour;
            material.FillMode = SCNFillMode.Lines;

            faceGeometry.Geometry.FirstMaterial = material;
        }

        public override void DidRemoveNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
        {
            if (anchor is ARFaceAnchor)
            {
            }
        }
    }
}