Feature/vision

.gitignore

@@ -288,4 +288,4 @@ FakesAssemblies/
 *.opt
 mixedreality-azure-samples.sln
 mixedreality-azure-samples.csproj
-mixedreality-azure-samples.Player.csproj
+mixedreality-azure-samples.Player.csproj

README.md

@@ -11,18 +11,22 @@ This project is organized into 3 key areas:
 Concise projects that demonstrate a capability quickly and don't have external dependencies. These samples can be downloaded independently and tested quickly with little or no server setup.
 
 Samples:
-- [Custom Vision with Windows ML in Hololens](https://github.com/meulta/mixedreality-azure-samples/tree/master/Standalone-Samples/WindowsML-CustomVision-Hololens)
+- [Custom Vision with Windows ML in HoloLens](https://github.com/meulta/mixedreality-azure-samples/tree/master/Standalone-Samples/WindowsML-CustomVision-Hololens)
+- [Unity with Custom Vision Treasure Hunt Demo for Hololens, Android, and iOS](https://github.com/Microsoft/mixedreality-azure-samples/tree/master/Standalone-Samples/VisionTreasureHunt)
+- [Unity with Cognitive Services Text-to-Speech](https://github.com/Microsoft/mixedreality-azure-samples/tree/master/Standalone-Samples/Unity-Text-to-Speech)
+- [Unity with Azure Storage](https://github.com/Microsoft/mixedreality-azure-samples/tree/master/Standalone-Samples/AzureStorageDemoUnity3D)
 
-
-### Solutions (Coming soon)
+### Solutions
 Also demonstrate a  capability but require additional setup. Solutions, for example, may require deploying an Azure workload like a bot, a function, or a database.
-
+
+Solutions:
+- [LUIS Caching Service](https://github.com/Microsoft/mixedreality-azure-samples/tree/master/Solutions/LUIS-CachingService): Reusable solution that showcases how to cache results from LUIS in Cognitive Services.
 
 ### Reference Architectures ###
 Are a *set of capabilities* that have all been designed to work together. For example, Speech Recognition may be used on its own but it is also designed to work with Language Understanding. 
 
 **Modules**
-- [LUIS for XR](Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/LUIS) - A powerful Natural Language replacement for voice commands.
+- [LUIS for XR](Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/LUIS): A powerful Natural Language replacement for voice commands.
 
 
 ## Contributing

Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/Common/Scripts/SecretHelper.cs

@@ -23,7 +23,6 @@
 // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 //
 
-using Microsoft.Cognitive.LUIS;
 using System;
 using System.Collections;
 using System.Collections.Generic;

Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/Vision/Prefabs/VisionObject.prefab

@@ -0,0 +1,63 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!1001 &100100000
+Prefab:
+  m_ObjectHideFlags: 1
+  serializedVersion: 2
+  m_Modification:
+    m_TransformParent: {fileID: 0}
+    m_Modifications: []
+    m_RemovedComponents: []
+  m_SourcePrefab: {fileID: 0}
+  m_RootGameObject: {fileID: 1033717228754310}
+  m_IsPrefabAsset: 1
+--- !u!1 &1033717228754310
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 4197923332607632}
+  - component: {fileID: 114818391866690084}
+  m_Layer: 0
+  m_Name: VisionObject
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!4 &4197923332607632
+Transform:
+  m_ObjectHideFlags: 1
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1033717228754310}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 0}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &114818391866690084
+MonoBehaviour:
+  m_ObjectHideFlags: 1
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInternal: {fileID: 100100000}
+  m_GameObject: {fileID: 1033717228754310}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 937c7d20bf0f1bb4f84585c327bd443e, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  useOffline: 0
+  useSDK: 0
+  useCustomVision: 1
+  predictionConfidenceThreshold: 0.5
+  customVisionURL: 
+  customVisionPredictionKey: 
+  customVisionProjectID: 
+  customVisionTrainingKey: 
+  visionURL: 
+  visionKey: 

Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/Vision/README.md

@@ -0,0 +1,27 @@
+<h1>Computer Vision and Custom Vision for XR</h1>
+<h3>Add image classification and object recognition to your AR application!</h3>
+
+<h2>How To Use</h2>
+(See the Vision Treasure Hunt demo in /Standalone-Samples for an example)
+
+1: Add the Unity Package "MR-Vision-x.x.x" to your Unity project.
+  </br> You'll find that in this Github Repository under MixedReality-Azure-Unity/UnityPackages
+
+2: Add the VisionObject prefab to your scene.
+   </br>That's under Assets/MixedRealityAzure/Vision after you've added the package.
+
+3: In the inspector for your scene's VisionObject, set your project information for Custom Vision, Computer Vision, or both!
+
+4: Get a reference to the VisionManager script component with something like:
+    </br>VisionManager visionManager = GetComponent<VisionManager>();
+
+5: Get an image as a byte[] formatted as a .png, .jpg, .bmp, or any other format accepted by CustomVision/ComputerVision.
+    </br> There are examples of how to do this using Hololens, using ARKit on iOS, or using ARCore for Android, in the Standalone Sample ""
+
+6: Send your image for prediction and get the result by calling:
+    </br>PredictionResult result = await visionManager.SendImageAsync(imageData); //imageData is the byte[] in this example
+
+7: Do whatever you like with the PredictionResult.
+    </br>It contains a list of Predictions for the image, where each Prediction has a name, confidence, and bounding box information if available.
+
+Now you have image classification in your app!

Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/Vision/Scripts/Prediction.cs

@@ -0,0 +1,10 @@
+//Class to store each prediction of a PredictionResult in a nice format
+namespace Microsoft.MR.Vision
+{
+    public class Prediction
+    {
+        public string name { get; set; }
+        public float confidence { get; set; }
+        public float[] boundingBoxInfo = null;// holds info about the bounding box, in fractions of the image (Left, Top, Width, Height)
+    }
+}

Reference-Architecture/Client/MixedReality-Azure-Unity/Assets/MixedRealityAzure/Vision/Scripts/PredictionResult.cs

@@ -0,0 +1,113 @@
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using UnityEngine;
+
+//Class to hold all the information of the results of an image prediction, for later use:
+namespace Microsoft.MR.Vision
+{
+    public class PredictionResult
+    {
+        public string jsonResultsString { get; set; }
+        public List<Prediction> predictions;//list to hold all predictions for a given image
+        public int indexOfHighestConfidence;
+        public float minConfidence;
+
+        public PredictionResult(float confidenceThreshold)
+        {
+            predictions = new List<Prediction>();
+            minConfidence = confidenceThreshold;
+        }
+
+        //turns the json string into a list of distinct Prediction objects:
+        public void JsonStringToPredictionList()
+        {
+            if (jsonResultsString != null)
+            {
+                JObject obj = JsonConvert.DeserializeObject<JObject>(this.jsonResultsString);
+
+                //get the predicted classes into a JToken:
+                List<JProperty> subProperties = obj.Properties().ToList();
+                bool isUsingCustomVision = true;
+                JProperty property = null;
+                try
+                {
+                    property = subProperties.Find(p => p.Name == "predictions");
+                    if (property == null)
+                    {
+                        property = subProperties.Find(p => p.Name == "categories");
+                        isUsingCustomVision = false;
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Debug.Log(ex.ToString());
+                }
+
+                JToken resultJToken = property.Value;
+
+                //make sure there are some classes:
+                if (resultJToken.Count() == 0)
+                {
+                    return;
+                }
+                //add the predictions to the prediction list:
+                //also keep track of which has highest confidence:
+                for (int i = 0; i < resultJToken.Count(); i++)
+                {
+                    if (isUsingCustomVision)
+                    {
+                        Prediction thisPrediction = new Prediction();
+                        thisPrediction.name = resultJToken[i].Value<string>("tagName");
+                        thisPrediction.confidence = resultJToken[i].Value<float>("probability");
+                        if (thisPrediction.confidence < minConfidence)
+                        {
+                            //skip this prediction:
+                            continue;
+                        }
+
+                        if (resultJToken[i].Value<JToken>("boundingBox") != null)
+                        {
+                            SetupBoundingBox(thisPrediction, resultJToken[i].Value<JToken>("boundingBox"));
+                        }
+                        predictions.Add(thisPrediction);
+                        if (thisPrediction.confidence > predictions[indexOfHighestConfidence].confidence)
+                        {
+                            indexOfHighestConfidence = predictions.Count - 1;//set this new prediction as highest confidence
+                        }
+                    }
+                    else
+                    {
+                        //using Computer Vision format:
+                        Prediction thisPrediction = new Prediction();
+                        thisPrediction.name = resultJToken[i].Value<string>("name");
+                        thisPrediction.confidence = resultJToken[i].Value<float>("score");
+                        if (thisPrediction.confidence < minConfidence)
+                        {
+                            //skip this prediction:
+                            continue;
+                        }
+                        predictions.Add(thisPrediction);
+                        if (thisPrediction.confidence > predictions[indexOfHighestConfidence].confidence)
+                        {
+                            indexOfHighestConfidence = predictions.Count - 1;//set this new prediction as highest confidence
+                        }
+                    }
+                }
+            }
+        }
+
+        //set up the corner points of bounding boxes, so that they can be shown later:
+        public void SetupBoundingBox(Prediction thisPrediction, JToken boundingBoxToken)
+        {
+            //box values are the fraction of the image:
+            thisPrediction.boundingBoxInfo = new float[4];
+            thisPrediction.boundingBoxInfo[0] = boundingBoxToken.Value<float>("left");
+            thisPrediction.boundingBoxInfo[1] = boundingBoxToken.Value<float>("top");
+            thisPrediction.boundingBoxInfo[2] = boundingBoxToken.Value<float>("width");
+            thisPrediction.boundingBoxInfo[3] = boundingBoxToken.Value<float>("height");
+        }
+    }
+}