Virtual table items

Description

Following on from the Gestures lesson and combining it with the Surface Plane Detection lesson i've created a proof of concept showing how virtual items can be arranged on a surface, moved around, rotated and scaled.

Video

Code

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using ARKit;
using Foundation;
using Newtonsoft.Json;
using SceneKit;
using UIKit;

namespace XamarinArkitSample
{
    public partial class ViewController : UIViewController, IARSCNViewDelegate
    {
        private readonly ARSCNView sceneView;
        private readonly IDictionary<NSUuid, SurfacePlaneNode> planeNodes = new Dictionary<NSUuid, SurfacePlaneNode>();
        public string unsplashAccessKey = "<YOUR_UNSPLASH_ACCESSKEY_HERE>";

        List<ImagePlaneNode> imagePlaneNodes = new List<ImagePlaneNode>();
        ConcurrentBag<UIImage> uiImages = new ConcurrentBag<UIImage>();
        private string searchTerm = "Pizza";

        private SurfacePlaneNode selectedSurface;

        public ViewController(IntPtr handle) : base(handle)
        {
            this.sceneView = new ARSCNView
            {   
                AutoenablesDefaultLighting = true,
                Delegate = this
            };

            this.View.AddSubview(this.sceneView);
        }

        public override void ViewDidLoad()
        {
            base.ViewDidLoad();

            this.sceneView.Frame = this.View.Frame;
        }

        public override void ViewDidDisappear(bool animated)
        {
            base.ViewDidDisappear(animated);

            this.sceneView.Session.Pause();
        }

        public override void ViewDidAppear(bool animated)
        {
            base.ViewDidAppear(animated);

            var configuration = new ARWorldTrackingConfiguration
            {
                AutoFocusEnabled = true,
                PlaneDetection = ARPlaneDetection.Horizontal,
                LightEstimationEnabled = true,
                WorldAlignment = ARWorldAlignment.Gravity,
            };

            this.sceneView.Session.Run(configuration, ARSessionRunOptions.ResetTracking | ARSessionRunOptions.RemoveExistingAnchors);

            // Setup gesture recognizers
            var panGesture = new UIPanGestureRecognizer(HandlePanGesture);
            this.sceneView.AddGestureRecognizer(panGesture);

            var rotateGesture = new UIRotationGestureRecognizer(HandleRotateGesture);
            this.sceneView.AddGestureRecognizer(rotateGesture);

            var pinchGesture = new UIPinchGestureRecognizer(HandlePinchGesture);
            this.sceneView.AddGestureRecognizer(pinchGesture);

            // Get images from UnSplash API
            Task.Run(async () =>
            {
                var imageUrls = await GetUrlsFromUnSplashApi(searchTerm, 20);

                foreach (var imageUrl in imageUrls)
                {
                    var taskA = LoadImage(imageUrl);

                    await taskA.ContinueWith(cw =>
                    {
                        var image = cw.Result;
                        uiImages.Add(image);
                        imagePlaneNodes.Add(new ImagePlaneNode(0.15f, 0.1f));
                    });
                }
            });
        }

        public async Task<UIImage> LoadImage(string url)
        {
            var httpClient = new WebClient();
            Task<byte[]> contentsTask = httpClient.DownloadDataTaskAsync(url);
            var contents = await contentsTask;

            return UIImage.LoadFromData(NSData.FromArray(contents));
        }

        public override void TouchesEnded(NSSet touches, UIEvent evt)
        {
            base.TouchesEnded(touches, evt);

            if (touches.AnyObject is UITouch touch)
            {
                var point = touch.LocationInView(this.sceneView);

                var hitTestOptions = new SCNHitTestOptions();

                var hits = this.sceneView.HitTest(point, hitTestOptions);
                var hit = hits.FirstOrDefault();

                if (hit == null)
                    return;

                var node = hit.Node;

                if (node == null)
                    return;

                if (selectedSurface != null)
                    return; 

                if(node is SurfacePlaneNode)
                {
                    // Set touched surface plane node as selected surface
                    selectedSurface = node as SurfacePlaneNode;

                    foreach(var item in planeNodes.Values)
                    {
                        if (selectedSurface.Guid != item.Guid)
                            item.RemoveFromParentNode();
                    }
                    
                    // Turn off plane detection
                    var configuration = new ARWorldTrackingConfiguration
                    {
                        AutoFocusEnabled = true,
                        PlaneDetection = ARPlaneDetection.None,
                        LightEstimationEnabled = true,
                        WorldAlignment = ARWorldAlignment.Gravity,
                    };

                    this.sceneView.Session.Run(configuration, ARSessionRunOptions.None);

                    var uiImageArray = uiImages.ToArray();

                    int counter = 0;

                    // Get image and place on table
                    for (int i = -5; i < 5; i++)
                    {
                        var x = selectedSurface.Position.X/2 + (0.03f*i);
                        var y = selectedSurface.Position.Y/2;
                        var z = selectedSurface.Position.Z + 0.1f + (0.001f * i);

                        var imageNode = imagePlaneNodes[counter];
                        imageNode.Position = new SCNVector3(x, y, z);

                        selectedSurface.AddChildNode(imageNode);
                        imageNode.UpdateImage(uiImageArray[counter]);
                        counter++;
                    }
                }
            }
        }
        
        [Export("renderer:didAddNode:forAnchor:")]
        public void DidAddNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
        {
            if (anchor is ARPlaneAnchor planeAnchor)
            {
                UIColor colour = UIColor.White;
                
                var planeNode = new SurfacePlaneNode(planeAnchor, colour);
                var angle = (float)(-Math.PI / 2);
                planeNode.EulerAngles = new SCNVector3(angle, 0, 0);

                node.AddChildNode(planeNode);
                this.planeNodes.Add(anchor.Identifier, planeNode);
            }
        }

        [Export("renderer:didRemoveNode:forAnchor:")]
        public void DidRemoveNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
        {
            if (anchor is ARPlaneAnchor)
            {
                if (this.planeNodes.ContainsKey(anchor.Identifier))
                {
                    this.planeNodes[anchor.Identifier].RemoveFromParentNode();
                    this.planeNodes.Remove(anchor.Identifier);
                }
            }
        }

        [Export("renderer:didUpdateNode:forAnchor:")]
        public void DidUpdateNode(ISCNSceneRenderer renderer, SCNNode node, ARAnchor anchor)
        {
            if (anchor is ARPlaneAnchor planeAnchor)
            {
                if(this.planeNodes.ContainsKey(anchor.Identifier))
                    this.planeNodes[anchor.Identifier].Update(planeAnchor);
            }
        }

        private async Task<string[]> GetUrlsFromUnSplashApi(string searchTerm, int perPage)
        {
            var client = new WebClient();
            var url = $"https://api.unsplash.com/search/photos?client_id={unsplashAccessKey}&page=1&per_page={perPage}&orientation=landscape&query={searchTerm}";

            var response = await client.DownloadStringTaskAsync(url);

            dynamic array = JsonConvert.DeserializeObject(response);

            var urls = new List<string>();
            foreach (var result in array["results"])
            {
                urls.Add(result.urls.small.ToString());
            }

            return urls.ToArray();
        }

        float currentAngleZ;
        float newAngleZ;

        private void HandlePinchGesture(UIPinchGestureRecognizer sender)
        {
            var areaPinched = sender.View as SCNView;
            var location = sender.LocationInView(areaPinched);
            var hitTestResults = areaPinched.HitTest(location, new SCNHitTestOptions());

            var hitTest = hitTestResults.FirstOrDefault();

            if (hitTest == null)
                return;

            var node = hitTest.Node;

            if (node is SurfacePlaneNode)
                return;

            var scaleX = (float)sender.Scale * node.Scale.X;
            var scaleY = (float)sender.Scale * node.Scale.Y;
            var scaleZ = (float)sender.Scale * node.Scale.Z;

            node.Scale = new SCNVector3(scaleX, scaleY, scaleZ);

            sender.Scale = 1;
        }

        private void HandleRotateGesture(UIRotationGestureRecognizer sender)
        {
            var areaTouched = sender.View as SCNView;
            var location = sender.LocationInView(areaTouched);
            var hitTestResults = areaTouched.HitTest(location, new SCNHitTestOptions());

            var hitTest = hitTestResults.FirstOrDefault();

            if (hitTest == null)
                return;

            var node = hitTest.Node;

            if (node is SurfacePlaneNode)
                return;

            newAngleZ = (float)(-sender.Rotation);
            newAngleZ += currentAngleZ;
            node.EulerAngles = new SCNVector3(node.EulerAngles.X, node.EulerAngles.Y, newAngleZ);
        }

        private void HandlePanGesture(UIPanGestureRecognizer sender)
        {
            var areaPanned = sender.View as SCNView;
            var location = sender.LocationInView(areaPanned);
            var hitTestResults = areaPanned.HitTest(location, new SCNHitTestOptions());

            var hitTest = hitTestResults.FirstOrDefault();

            if (hitTest == null)
                return;

            var node = hitTest.Node;

            if (node is SurfacePlaneNode)
                return;

            if (sender.State == UIGestureRecognizerState.Changed)
            {
                var translate = sender.TranslationInView(areaPanned);

                // Only allow movement vertically or horizontally
                node.LocalTranslate(new SCNVector3((float)translate.X / 10000f, (float)-translate.Y / 10000, 0.0f));
            }
        }
    }

    internal class SurfacePlaneNode : SCNNode
    {
        private readonly SCNPlane planeGeometry;
        public Guid Guid { get; set; }

        public SurfacePlaneNode(ARPlaneAnchor planeAnchor, UIColor colour)
        {
            Guid = Guid.NewGuid();
            Geometry = (planeGeometry = CreateGeometry(planeAnchor, colour));
        }

        public void Update(ARPlaneAnchor planeAnchor)
        {
            planeGeometry.Width = planeAnchor.Extent.X;
            planeGeometry.Height = planeAnchor.Extent.Z;

            Position = new SCNVector3(
                planeAnchor.Center.X,
                planeAnchor.Center.Y,
                planeAnchor.Center.Z);
        }

        private static SCNPlane CreateGeometry(ARPlaneAnchor planeAnchor, UIColor colour)
        {
            var material = new SCNMaterial();
            material.Diffuse.Contents = colour;
            material.DoubleSided = true;
            material.Transparency = 0.3f;

            var geometry = SCNPlane.Create(planeAnchor.Extent.X, planeAnchor.Extent.Z);
            geometry.Materials = new[] { material };

            return geometry;
        }
    }

    public class ImagePlaneNode : SCNNode
    {
        public ImagePlaneNode(float width, float height)
        {
            Geometry = CreateGeometry(width, height);
        }

        private static SCNGeometry CreateGeometry(float width, float height)
        {
            var material = new SCNMaterial();
            material.Diffuse.Contents = UIColor.White;
            material.DoubleSided = true;

            var geometry = SCNPlane.Create(width, height);
            geometry.Materials = new[] { material };

            return geometry;
        }

        internal void UpdateImage(UIImage uIImage)
        {
            Geometry.FirstMaterial.Diffuse.Contents = uIImage;
        }
    }
}